diff --git a/Core/nbproject/project.xml b/Core/nbproject/project.xml index 597af6c073..d3ac663a28 100644 --- a/Core/nbproject/project.xml +++ b/Core/nbproject/project.xml @@ -232,7 +232,8 @@ - 1.0 + 1 + 23 diff --git a/Core/src/org/sleuthkit/autopsy/core/UserPreferences.java b/Core/src/org/sleuthkit/autopsy/core/UserPreferences.java index 3dc28a49be..08c6f8f6d4 100644 --- a/Core/src/org/sleuthkit/autopsy/core/UserPreferences.java +++ b/Core/src/org/sleuthkit/autopsy/core/UserPreferences.java @@ -62,6 +62,7 @@ public final class UserPreferences { private static final String MESSAGE_SERVICE_HOST = "MessageServiceHost"; //NON-NLS private static final String MESSAGE_SERVICE_PORT = "MessageServicePort"; //NON-NLS public static final String TEXT_TRANSLATOR_NAME = "TextTranslatorName"; + public static final String OCR_TRANSLATION_ENABLED = "OcrTranslationEnabled"; public static final String PROCESS_TIME_OUT_ENABLED = "ProcessTimeOutEnabled"; //NON-NLS public static final String PROCESS_TIME_OUT_HOURS = "ProcessTimeOutHours"; //NON-NLS private static final int DEFAULT_PROCESS_TIMEOUT_HR = 60; @@ -347,6 +348,14 @@ public final class UserPreferences { public static String getTextTranslatorName() { return preferences.get(TEXT_TRANSLATOR_NAME, null); } + + public static void setUseOcrInTranslation(boolean enableOcr) { + preferences.putBoolean(OCR_TRANSLATION_ENABLED, enableOcr); + } + + public static boolean getUseOcrInTranslation() { + return preferences.getBoolean(OCR_TRANSLATION_ENABLED, true); + } /** * Persists message service connection info. @@ -623,4 +632,4 @@ public final class UserPreferences { return Paths.get(UserMachinePreferences.getBaseTempDirectory(), getAppName()) .toAbsolutePath().toString(); } -} +} \ No newline at end of file diff --git a/Core/src/org/sleuthkit/autopsy/corecomponents/AutoWrappingJTextPane.java b/Core/src/org/sleuthkit/autopsy/corecomponents/AutoWrappingJTextPane.java index 8b739b8dcf..1e4c1f4c63 100755 --- a/Core/src/org/sleuthkit/autopsy/corecomponents/AutoWrappingJTextPane.java +++ b/Core/src/org/sleuthkit/autopsy/corecomponents/AutoWrappingJTextPane.java @@ -27,6 +27,7 @@ import javax.swing.text.ViewFactory; import javax.swing.text.html.HTMLEditorKit; import javax.swing.text.html.InlineView; import javax.swing.text.html.ParagraphView; +import org.sleuthkit.autopsy.coreutils.EscapeUtil; /** * JTextPane extension that auto wraps input text using an HTMLEditorKit trick. @@ -98,6 +99,6 @@ public class AutoWrappingJTextPane extends JTextPane { @Override public void setText(String text) { - super.setText("
" + text + "
"); + super.setText("
" + EscapeUtil.escapeHtml(text) + "
"); } } diff --git a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/EmbeddedFileExtractorIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/EmbeddedFileExtractorIngestModule.java index fd833b59a7..dc9e7c6d1d 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/EmbeddedFileExtractorIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/EmbeddedFileExtractorIngestModule.java @@ -20,7 +20,12 @@ package org.sleuthkit.autopsy.modules.embeddedfileextractor; import java.io.File; import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; import java.util.concurrent.ConcurrentHashMap; +import javax.imageio.ImageIO; +import javax.imageio.spi.IIORegistry; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.datamodel.AbstractFile; @@ -122,8 +127,53 @@ public final class EmbeddedFileExtractorIngestModule extends FileIngestModuleAda } catch (NoCurrentCaseException ex) { throw new IngestModuleException(Bundle.EmbeddedFileExtractorIngestModule_UnableToGetMSOfficeExtractor_errMsg(), ex); } + + /** + * Initialize Java's Image I/O API so that image reading and writing + * (needed for image extraction) happens consistently through the + * same providers. See JIRA-6951 for more details. + */ + initializeImageIO(); } + + /** + * Initializes the ImageIO API and sorts the providers for + * deterministic image reading and writing. + */ + private void initializeImageIO() { + ImageIO.scanForPlugins(); + + // Sift through each registry category and sort category providers by + // their canonical class name. + IIORegistry pluginRegistry = IIORegistry.getDefaultInstance(); + Iterator> categories = pluginRegistry.getCategories(); + while(categories.hasNext()) { + sortPluginsInCategory(pluginRegistry, categories.next()); + } + } + + /** + * Sorts all ImageIO SPI providers by their class name. + */ + private void sortPluginsInCategory(IIORegistry pluginRegistry, Class category) { + Iterator serviceProviderIter = pluginRegistry.getServiceProviders(category, false); + ArrayList providers = new ArrayList<>(); + while (serviceProviderIter.hasNext()) { + providers.add(serviceProviderIter.next()); + } + Collections.sort(providers, (first, second) -> { + return first.getClass().getCanonicalName().compareToIgnoreCase(second.getClass().getCanonicalName()); + }); + for(int i = 0; i < providers.size() - 1; i++) { + for(int j = i + 1; j < providers.size(); j++) { + // The registry only accepts pairwise orderings. To guarantee a + // total order, all pairs need to be exhausted. + pluginRegistry.setOrdering(category, providers.get(i), + providers.get(j)); + } + } + } @Override public ProcessResult process(AbstractFile abstractFile) { diff --git a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/PDFAttachmentExtractor.java b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/PDFAttachmentExtractor.java index 15b1f1beb7..9cea0f63e7 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/PDFAttachmentExtractor.java +++ b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/PDFAttachmentExtractor.java @@ -123,7 +123,7 @@ final class PDFAttachmentExtractor { @Override public void parseEmbedded(InputStream in, ContentHandler ch, Metadata mtdt, boolean bln) throws SAXException, IOException { //Resource naming scheme is used internally in autopsy, therefore we can guarentee uniqueness. - String uniqueExtractedName = parentID + "_attch_" + attachmentCount++; //NON-NLS + String uniqueExtractedName = "extract_" + attachmentCount++; //NON-NLS String name = mtdt.get(Metadata.RESOURCE_NAME_KEY); String ext = FilenameUtils.getExtension(name); diff --git a/Core/src/org/sleuthkit/autopsy/textextractors/TikaTextExtractor.java b/Core/src/org/sleuthkit/autopsy/textextractors/TikaTextExtractor.java index a8bf0591fb..ee5b8d8fc3 100644 --- a/Core/src/org/sleuthkit/autopsy/textextractors/TikaTextExtractor.java +++ b/Core/src/org/sleuthkit/autopsy/textextractors/TikaTextExtractor.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2011-2019 Basis Technology Corp. + * Copyright 2011-2020 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -46,7 +46,6 @@ import org.apache.tika.Tika; import org.apache.tika.exception.TikaException; import org.apache.tika.metadata.Metadata; import org.apache.tika.parser.AutoDetectParser; -import org.apache.tika.parser.EmptyParser; import org.apache.tika.parser.ParseContext; import org.apache.tika.parser.Parser; import org.apache.tika.parser.ParsingReader; @@ -72,6 +71,9 @@ import org.xml.sax.ContentHandler; import org.xml.sax.SAXException; import org.xml.sax.helpers.DefaultHandler; import com.google.common.collect.ImmutableMap; +import java.io.InputStreamReader; +import java.nio.charset.Charset; +import org.apache.tika.parser.pdf.PDFParserConfig.OCR_STRATEGY; /** * Extracts text from Tika supported content. Protects against Tika parser hangs @@ -126,16 +128,6 @@ final class TikaTextExtractor implements TextExtractor { "application/x-z", //NON-NLS "application/x-compress"); //NON-NLS - //Tika should ignore types with embedded files that can be handled by the unpacking modules - private static final List EMBEDDED_FILE_MIME_TYPES - = ImmutableList.of("application/msword", //NON-NLS - "application/vnd.openxmlformats-officedocument.wordprocessingml.document", //NON-NLS - "application/vnd.ms-powerpoint", //NON-NLS - "application/vnd.openxmlformats-officedocument.presentationml.presentation", //NON-NLS - "application/vnd.ms-excel", //NON-NLS - "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", //NON-NLS - "application/pdf"); //NON-NLS - // Used to log to the tika file that is why it uses the java.util.logging.logger class instead of the Autopsy one private static final java.util.logging.Logger TIKA_LOGGER = java.util.logging.Logger.getLogger("Tika"); //NON-NLS private static final Logger AUTOPSY_LOGGER = Logger.getLogger(TikaTextExtractor.class.getName()); @@ -193,52 +185,31 @@ final class TikaTextExtractor implements TextExtractor { */ @Override public Reader getReader() throws InitReaderException { - InputStream stream = null; - - ParseContext parseContext = new ParseContext(); - - //Disable appending embedded file text to output for EFE supported types - //JIRA-4975 - if(content instanceof AbstractFile && EMBEDDED_FILE_MIME_TYPES.contains(((AbstractFile)content).getMIMEType())) { - parseContext.set(Parser.class, new EmptyParser()); - } else { - parseContext.set(Parser.class, parser); + if (!this.isSupported()) { + throw new InitReaderException("Content is not supported"); } - if (ocrEnabled() && content instanceof AbstractFile) { - AbstractFile file = ((AbstractFile) content); - //Run OCR on images with Tesseract directly. - if (file.getMIMEType().toLowerCase().startsWith("image/")) { - stream = performOCR(file); - } else { - //Otherwise, go through Tika for PDFs so that it can - //extract images and run Tesseract on them. - PDFParserConfig pdfConfig = new PDFParserConfig(); + // Only abstract files are supported, see isSupported() + final AbstractFile file = ((AbstractFile) content); + // This mime type must be non-null, see isSupported() + final String mimeType = file.getMIMEType(); - // Extracting the inline images and letting Tesseract run on each inline image. - // https://wiki.apache.org/tika/PDFParser%20%28Apache%20PDFBox%29 - // https://tika.apache.org/1.7/api/org/apache/tika/parser/pdf/PDFParserConfig.html - pdfConfig.setExtractInlineImages(true); - // Multiple pages within a PDF file might refer to the same underlying image. - pdfConfig.setExtractUniqueInlineImagesOnly(true); - parseContext.set(PDFParserConfig.class, pdfConfig); - - // Configure Tesseract parser to perform OCR - TesseractOCRConfig ocrConfig = new TesseractOCRConfig(); - String tesseractFolder = TESSERACT_PATH.getParent(); - ocrConfig.setTesseractPath(tesseractFolder); - - ocrConfig.setLanguage(languagePacks); - ocrConfig.setTessdataPath(PlatformUtil.getOcrLanguagePacksPath()); - parseContext.set(TesseractOCRConfig.class, ocrConfig); - - stream = new ReadContentInputStream(content); - } - } else { - stream = new ReadContentInputStream(content); + // Handle images seperately so the OCR task can be cancelled. + // See JIRA-4519 for the need to have cancellation in the UI and ingest. + if (ocrEnabled() && mimeType.toLowerCase().startsWith("image/")) { + InputStream imageOcrStream = performOCR(file); + return new InputStreamReader(imageOcrStream, Charset.forName("UTF-8")); } - Metadata metadata = new Metadata(); + // Set up Tika + final InputStream stream = new ReadContentInputStream(content); + final ParseContext parseContext = new ParseContext(); + + // Documents can contain other documents. By adding + // the parser back into the context, Tika will recursively + // parse embedded documents. + parseContext.set(Parser.class, parser); + // Use the more memory efficient Tika SAX parsers for DOCX and // PPTX files (it already uses SAX for XLSX). OfficeParserConfig officeParserConfig = new OfficeParserConfig(); @@ -246,6 +217,30 @@ final class TikaTextExtractor implements TextExtractor { officeParserConfig.setUseSAXDocxExtractor(true); parseContext.set(OfficeParserConfig.class, officeParserConfig); + if (ocrEnabled()) { + // Configure OCR for Tika if it chooses to run OCR + // during extraction + TesseractOCRConfig ocrConfig = new TesseractOCRConfig(); + String tesseractFolder = TESSERACT_PATH.getParent(); + ocrConfig.setTesseractPath(tesseractFolder); + ocrConfig.setLanguage(languagePacks); + ocrConfig.setTessdataPath(PlatformUtil.getOcrLanguagePacksPath()); + parseContext.set(TesseractOCRConfig.class, ocrConfig); + + // Configure how Tika handles OCRing PDFs + PDFParserConfig pdfConfig = new PDFParserConfig(); + + // This stategy tries to pick between OCRing a page in the + // PDF and doing text extraction. It makes this choice by + // first running text extraction and then counting characters. + // If there are too few characters or too many unmapped + // unicode characters, it'll run the entire page through OCR + // and take that output instead. See JIRA-6938 + pdfConfig.setOcrStrategy(OCR_STRATEGY.AUTO); + parseContext.set(PDFParserConfig.class, pdfConfig); + } + + Metadata metadata = new Metadata(); //Make the creation of a TikaReader a cancellable future in case it takes too long Future future = executorService.submit( new GetTikaReader(parser, stream, metadata, parseContext)); @@ -568,4 +563,4 @@ final class TikaTextExtractor implements TextExtractor { return reader; } } -} +} \ No newline at end of file diff --git a/Core/src/org/sleuthkit/autopsy/texttranslation/Bundle.properties b/Core/src/org/sleuthkit/autopsy/texttranslation/Bundle.properties index 5fcf80851c..bd321cf492 100755 --- a/Core/src/org/sleuthkit/autopsy/texttranslation/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/texttranslation/Bundle.properties @@ -6,3 +6,4 @@ TranslationOptionsPanelController.moduleErr.msg=A module caused an error listeni TranslationContentPanel.showLabel.text=Show: TranslationOptionsPanel.translationServiceLabel.text=Text translator: TranslationOptionsPanel.translationOptionsDescription.text=Configure a 3rd party text translation service to enable text and file name translation. +TranslationOptionsPanel.enableOcrCheckBox.text=Enable Optical Character Recognition (OCR) in the translation content viewer diff --git a/Core/src/org/sleuthkit/autopsy/texttranslation/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/texttranslation/Bundle.properties-MERGED index 665c07c36f..2da3aab4a4 100755 --- a/Core/src/org/sleuthkit/autopsy/texttranslation/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/texttranslation/Bundle.properties-MERGED @@ -10,3 +10,4 @@ TranslationOptionsPanelController.moduleErr.msg=A module caused an error listeni TranslationContentPanel.showLabel.text=Show: TranslationOptionsPanel.translationServiceLabel.text=Text translator: TranslationOptionsPanel.translationOptionsDescription.text=Configure a 3rd party text translation service to enable text and file name translation. +TranslationOptionsPanel.enableOcrCheckBox.text=Enable Optical Character Recognition (OCR) in the translation content viewer diff --git a/Core/src/org/sleuthkit/autopsy/texttranslation/TranslationOptionsPanel.form b/Core/src/org/sleuthkit/autopsy/texttranslation/TranslationOptionsPanel.form index 52cd0fb65c..aadca0381f 100644 --- a/Core/src/org/sleuthkit/autopsy/texttranslation/TranslationOptionsPanel.form +++ b/Core/src/org/sleuthkit/autopsy/texttranslation/TranslationOptionsPanel.form @@ -16,17 +16,23 @@ + + - - - + + + + + + + + - @@ -42,9 +48,13 @@ - - - + + + + + + + @@ -76,5 +86,17 @@ + + + + + + + + + + + + - + \ No newline at end of file diff --git a/Core/src/org/sleuthkit/autopsy/texttranslation/TranslationOptionsPanel.java b/Core/src/org/sleuthkit/autopsy/texttranslation/TranslationOptionsPanel.java index 3c9ee7dc7b..9fdb95b2b1 100644 --- a/Core/src/org/sleuthkit/autopsy/texttranslation/TranslationOptionsPanel.java +++ b/Core/src/org/sleuthkit/autopsy/texttranslation/TranslationOptionsPanel.java @@ -111,6 +111,7 @@ final class TranslationOptionsPanel extends javax.swing.JPanel { } translatorComboBox.setSelectedItem(currentSelection); loadSelectedPanelSettings(); + enableOcrCheckBox.setSelected(UserPreferences.getUseOcrInTranslation()); } /** @@ -128,6 +129,8 @@ final class TranslationOptionsPanel extends javax.swing.JPanel { logger.log(Level.WARNING, "Unable to save settings for TextTranslator named: " + currentSelection, ex); } } + // Save whether OCR is enabled in the content viewer + UserPreferences.setUseOcrInTranslation(enableOcrCheckBox.isSelected()); } @@ -144,6 +147,8 @@ final class TranslationOptionsPanel extends javax.swing.JPanel { translationServiceLabel = new javax.swing.JLabel(); translationServicePanel = new javax.swing.JPanel(); translationOptionsDescription = new javax.swing.JLabel(); + jSeparator1 = new javax.swing.JSeparator(); + enableOcrCheckBox = new javax.swing.JCheckBox(); translatorComboBox.addActionListener(new java.awt.event.ActionListener() { public void actionPerformed(java.awt.event.ActionEvent evt) { @@ -157,20 +162,31 @@ final class TranslationOptionsPanel extends javax.swing.JPanel { org.openide.awt.Mnemonics.setLocalizedText(translationOptionsDescription, org.openide.util.NbBundle.getMessage(TranslationOptionsPanel.class, "TranslationOptionsPanel.translationOptionsDescription.text")); // NOI18N + org.openide.awt.Mnemonics.setLocalizedText(enableOcrCheckBox, org.openide.util.NbBundle.getMessage(TranslationOptionsPanel.class, "TranslationOptionsPanel.enableOcrCheckBox.text")); // NOI18N + enableOcrCheckBox.addActionListener(new java.awt.event.ActionListener() { + public void actionPerformed(java.awt.event.ActionEvent evt) { + enableOcrCheckBoxActionPerformed(evt); + } + }); + javax.swing.GroupLayout layout = new javax.swing.GroupLayout(this); this.setLayout(layout); layout.setHorizontalGroup( layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addComponent(jSeparator1) .addGroup(layout.createSequentialGroup() .addContainerGap() .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addComponent(translationServicePanel, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) + .addComponent(translationOptionsDescription, javax.swing.GroupLayout.DEFAULT_SIZE, 462, Short.MAX_VALUE) .addGroup(layout.createSequentialGroup() - .addComponent(translationServiceLabel) - .addGap(10, 10, 10) - .addComponent(translatorComboBox, javax.swing.GroupLayout.PREFERRED_SIZE, 214, javax.swing.GroupLayout.PREFERRED_SIZE) - .addGap(0, 0, Short.MAX_VALUE)) - .addComponent(translationOptionsDescription, javax.swing.GroupLayout.PREFERRED_SIZE, 462, Short.MAX_VALUE)) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addGroup(layout.createSequentialGroup() + .addComponent(translationServiceLabel) + .addGap(10, 10, 10) + .addComponent(translatorComboBox, javax.swing.GroupLayout.PREFERRED_SIZE, 214, javax.swing.GroupLayout.PREFERRED_SIZE)) + .addComponent(enableOcrCheckBox)) + .addGap(0, 0, Short.MAX_VALUE))) .addContainerGap()) ); layout.setVerticalGroup( @@ -183,8 +199,12 @@ final class TranslationOptionsPanel extends javax.swing.JPanel { .addComponent(translatorComboBox, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE) .addComponent(translationServiceLabel)) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) - .addComponent(translationServicePanel, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) - .addContainerGap()) + .addComponent(translationServicePanel, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) + .addComponent(jSeparator1, javax.swing.GroupLayout.PREFERRED_SIZE, 10, javax.swing.GroupLayout.PREFERRED_SIZE) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addComponent(enableOcrCheckBox) + .addContainerGap(javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)) ); }// //GEN-END:initComponents @@ -192,12 +212,18 @@ final class TranslationOptionsPanel extends javax.swing.JPanel { updatePanel(); }//GEN-LAST:event_translatorComboBoxActionPerformed + private void enableOcrCheckBoxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_enableOcrCheckBoxActionPerformed + controller.changed(); + }//GEN-LAST:event_enableOcrCheckBoxActionPerformed + // Variables declaration - do not modify//GEN-BEGIN:variables + private javax.swing.JCheckBox enableOcrCheckBox; + private javax.swing.JSeparator jSeparator1; private javax.swing.JLabel translationOptionsDescription; private javax.swing.JLabel translationServiceLabel; private javax.swing.JPanel translationServicePanel; private javax.swing.JComboBox translatorComboBox; // End of variables declaration//GEN-END:variables -} +} \ No newline at end of file diff --git a/Core/src/org/sleuthkit/autopsy/texttranslation/ui/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/texttranslation/ui/Bundle.properties-MERGED index 802b117c35..c87d7ae238 100644 --- a/Core/src/org/sleuthkit/autopsy/texttranslation/ui/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/texttranslation/ui/Bundle.properties-MERGED @@ -8,6 +8,7 @@ TranslatedContentViewer.errorExtractingText=An error occurred while extracting t TranslatedContentViewer.extractingText=Extracting text, please wait... TranslatedContentViewer.fileHasNoText=File has no text. TranslatedContentViewer.noServiceProvider=The machine translation software was not found. +TranslatedContentViewer.ocrNotEnabled=OCR is not enabled. To change, go to Tools->Options->Machine Translation TranslatedContentViewer.translatingText=Translating text, please wait... # {0} - exception message TranslatedContentViewer.translationException=An error occurred while translating the text ({0}). diff --git a/Core/src/org/sleuthkit/autopsy/texttranslation/ui/TranslatedTextViewer.java b/Core/src/org/sleuthkit/autopsy/texttranslation/ui/TranslatedTextViewer.java index ab0b9de083..6b04427e92 100644 --- a/Core/src/org/sleuthkit/autopsy/texttranslation/ui/TranslatedTextViewer.java +++ b/Core/src/org/sleuthkit/autopsy/texttranslation/ui/TranslatedTextViewer.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2019 Basis Technology Corp. + * Copyright 2020 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -38,16 +38,15 @@ import org.sleuthkit.datamodel.AbstractFile; import org.openide.util.Lookup; import org.openide.util.NbBundle; import org.openide.util.lookup.Lookups; -import org.sleuthkit.autopsy.corecomponents.DataContentViewerUtility; import org.sleuthkit.autopsy.coreutils.ExecUtil.ProcessTerminator; import org.sleuthkit.autopsy.textextractors.TextExtractor; import org.sleuthkit.autopsy.textextractors.TextExtractorFactory; import org.sleuthkit.autopsy.textextractors.configs.ImageConfig; import org.sleuthkit.autopsy.texttranslation.TextTranslationService; -import org.sleuthkit.datamodel.Content; import java.util.List; import java.util.logging.Level; import javax.swing.SwingUtilities; +import org.sleuthkit.autopsy.core.UserPreferences; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.PlatformUtil; import org.sleuthkit.autopsy.texttranslation.ui.TranslationContentPanel.DisplayDropdownOptions; @@ -60,8 +59,6 @@ public final class TranslatedTextViewer implements TextViewer { private static final Logger logger = Logger.getLogger(TranslatedTextViewer.class.getName()); - private static final boolean OCR_ENABLED = true; - private static final boolean OCR_DISABLED = false; private static final int MAX_EXTRACT_SIZE_BYTES = 25600; private static final List INSTALLED_LANGUAGE_PACKS = PlatformUtil.getOcrLanguagePacks(); private final TranslationContentPanel panel = new TranslationContentPanel(); @@ -81,15 +78,10 @@ public final class TranslatedTextViewer implements TextViewer { SelectionChangeListener displayDropDownListener = new DisplayDropDownChangeListener(); panel.addDisplayTextActionListener(displayDropDownListener); panel.addOcrDropDownActionListener(new OCRDropdownChangeListener()); - Content source = DataContentViewerUtility.getDefaultContent(node); - - if (source instanceof AbstractFile) { - boolean isImage = ((AbstractFile) source).getMIMEType().toLowerCase().startsWith("image/"); - if (isImage) { - panel.enableOCRSelection(OCR_ENABLED); - panel.addLanguagePackNames(INSTALLED_LANGUAGE_PACKS); - } + if (UserPreferences.getUseOcrInTranslation()) { + panel.addLanguagePackNames(INSTALLED_LANGUAGE_PACKS); } + panel.enableOCRSelection(UserPreferences.getUseOcrInTranslation()); int payloadMaxInKB = TextTranslationService.getInstance().getMaxTextChars() / 1000; panel.setWarningLabelMsg(String.format(Bundle.TranslatedTextViewer_maxPayloadSize(), payloadMaxInKB)); @@ -201,14 +193,16 @@ public final class TranslatedTextViewer implements TextViewer { * @throws * org.sleuthkit.autopsy.textextractors.TextExtractor.InitReaderException */ + @NbBundle.Messages({ + "TranslatedContentViewer.ocrNotEnabled=OCR is not enabled. To change, go to Tools->Options->Machine Translation", + }) private String getFileText(AbstractFile file) throws IOException, InterruptedException, TextExtractor.InitReaderException { final boolean isImage = file.getMIMEType().toLowerCase().startsWith("image/"); // NON-NLS - String result; - if (isImage) { - result = extractText(file, OCR_ENABLED); - } else { - result = extractText(file, OCR_DISABLED); + if (isImage && ! UserPreferences.getUseOcrInTranslation()) { + return Bundle.TranslatedContentViewer_ocrNotEnabled(); } + + String result = extractText(file, UserPreferences.getUseOcrInTranslation()); //Correct for UTF-8 byte[] resultInUTF8Bytes = result.getBytes("UTF8"); @@ -363,4 +357,4 @@ public final class TranslatedTextViewer implements TextViewer { return panel.getSelectedOcrLanguagePack(); } } -} +} \ No newline at end of file diff --git a/Core/test/qa-functional/src/org/sleuthkit/autopsy/README.txt b/Core/test/qa-functional/src/org/sleuthkit/autopsy/README.txt new file mode 100644 index 0000000000..139b74c67f --- /dev/null +++ b/Core/test/qa-functional/src/org/sleuthkit/autopsy/README.txt @@ -0,0 +1 @@ +Netbeans platform does not properly scope classloaders while running qa-functional test code. The result is that NoClassDefError's occur in instances where an external jar (i.e. importing a class from common-io) is referenced in test code and the same external jar is referenced in multiple NBM's. Importing from external jars in qa-functional should be avoided. See jira issue 6954 for more information. \ No newline at end of file diff --git a/Core/test/qa-functional/src/org/sleuthkit/autopsy/centralrepository/datamodel/CentralRepoAccountsTest.java b/Core/test/qa-functional/src/org/sleuthkit/autopsy/centralrepository/datamodel/CentralRepoAccountsTest.java index ff8bd34250..a87c2959ba 100755 --- a/Core/test/qa-functional/src/org/sleuthkit/autopsy/centralrepository/datamodel/CentralRepoAccountsTest.java +++ b/Core/test/qa-functional/src/org/sleuthkit/autopsy/centralrepository/datamodel/CentralRepoAccountsTest.java @@ -25,11 +25,10 @@ import java.nio.file.Paths; import junit.framework.Assert; import junit.framework.TestCase; import junit.framework.Test; -import org.apache.commons.io.FileUtils; import org.netbeans.junit.NbModuleSuite; -import org.openide.util.Exceptions; import org.sleuthkit.autopsy.centralrepository.datamodel.CentralRepoAccount.CentralRepoAccountType; +import org.sleuthkit.autopsy.coreutils.FileUtil; import org.sleuthkit.datamodel.Account; import org.sleuthkit.datamodel.InvalidAccountIDException; @@ -95,7 +94,8 @@ public class CentralRepoAccountsTest extends TestCase { if (CentralRepository.isEnabled()) { CentralRepository.getInstance().shutdownConnections(); } - FileUtils.deleteDirectory(testDirectory.toFile()); + + FileUtil.deleteDir(testDirectory.toFile()); } public void testPredefinedAccountTypes() { diff --git a/Core/test/qa-functional/src/org/sleuthkit/autopsy/centralrepository/datamodel/CentralRepoDatamodelTest.java b/Core/test/qa-functional/src/org/sleuthkit/autopsy/centralrepository/datamodel/CentralRepoDatamodelTest.java index bf71e5f6a7..357d5e0641 100755 --- a/Core/test/qa-functional/src/org/sleuthkit/autopsy/centralrepository/datamodel/CentralRepoDatamodelTest.java +++ b/Core/test/qa-functional/src/org/sleuthkit/autopsy/centralrepository/datamodel/CentralRepoDatamodelTest.java @@ -35,7 +35,6 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; import junit.framework.Test; import junit.framework.TestCase; -import org.apache.commons.io.FileUtils; import org.netbeans.junit.NbModuleSuite; import org.openide.util.Exceptions; import junit.framework.Assert; @@ -47,6 +46,7 @@ import org.sleuthkit.datamodel.TskData; import static junit.framework.Assert.assertFalse; import static junit.framework.Assert.assertTrue; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; +import org.sleuthkit.autopsy.coreutils.FileUtil; /** * Functional tests for the Central Repository data model. @@ -100,8 +100,8 @@ public class CentralRepoDatamodelTest extends TestCase { if (CentralRepository.isEnabled()) { CentralRepository.getInstance().shutdownConnections(); } - FileUtils.deleteDirectory(testDirectory.toFile()); - } catch (IOException | CentralRepoException ex) { + FileUtil.deleteDir(testDirectory.toFile()); + } catch (CentralRepoException ex) { Assert.fail(ex.getMessage()); } } @@ -194,8 +194,8 @@ public class CentralRepoDatamodelTest extends TestCase { if (CentralRepository.isEnabled()) { CentralRepository.getInstance().shutdownConnections(); } - FileUtils.deleteDirectory(testDirectory.toFile()); - } catch (CentralRepoException | IOException ex) { + FileUtil.deleteDir(testDirectory.toFile()); + } catch (CentralRepoException ex) { Exceptions.printStackTrace(ex); Assert.fail(ex.getMessage()); } diff --git a/Core/test/qa-functional/src/org/sleuthkit/autopsy/centralrepository/datamodel/CentralRepoPersonasTest.java b/Core/test/qa-functional/src/org/sleuthkit/autopsy/centralrepository/datamodel/CentralRepoPersonasTest.java index bbe5ae58f5..60636e5f54 100644 --- a/Core/test/qa-functional/src/org/sleuthkit/autopsy/centralrepository/datamodel/CentralRepoPersonasTest.java +++ b/Core/test/qa-functional/src/org/sleuthkit/autopsy/centralrepository/datamodel/CentralRepoPersonasTest.java @@ -28,10 +28,10 @@ import junit.framework.Assert; import static junit.framework.Assert.assertTrue; import junit.framework.TestCase; import junit.framework.Test; -import org.apache.commons.io.FileUtils; import org.netbeans.junit.NbModuleSuite; import org.openide.util.Exceptions; +import org.sleuthkit.autopsy.coreutils.FileUtil; import org.sleuthkit.datamodel.Account; import org.sleuthkit.datamodel.InvalidAccountIDException; import org.sleuthkit.datamodel.TskData; @@ -257,12 +257,12 @@ public class CentralRepoPersonasTest extends TestCase { // This function is run after every test, NOT after the entire collection of // tests defined in the class are run. @Override - public void tearDown() throws CentralRepoException, IOException { + public void tearDown() throws CentralRepoException { // Close and delete the test case and central repo db if (CentralRepository.isEnabled()) { CentralRepository.getInstance().shutdownConnections(); } - FileUtils.deleteDirectory(testDirectory.toFile()); + FileUtil.deleteDir(testDirectory.toFile()); } diff --git a/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonpropertiessearch/InterCaseTestUtils.java b/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonpropertiessearch/InterCaseTestUtils.java index 85fe4d91c7..014b08fe84 100644 --- a/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonpropertiessearch/InterCaseTestUtils.java +++ b/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonpropertiessearch/InterCaseTestUtils.java @@ -28,7 +28,6 @@ import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.Map; -import org.apache.commons.io.FileUtils; import org.netbeans.junit.NbTestCase; import org.openide.util.Exceptions; import org.sleuthkit.autopsy.casemodule.Case; @@ -63,6 +62,7 @@ import org.sleuthkit.autopsy.modules.vmextractor.VMExtractorIngestModuleFactory; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.autopsy.centralrepository.datamodel.CentralRepository; import org.sleuthkit.autopsy.centralrepository.datamodel.RdbmsCentralRepoFactory; +import org.sleuthkit.autopsy.coreutils.FileUtil; import org.sleuthkit.autopsy.modules.pictureanalyzer.PictureAnalyzerIngestModuleFactory; /** @@ -247,8 +247,8 @@ class InterCaseTestUtils { if (CentralRepository.isEnabled()) { CentralRepository.getInstance().shutdownConnections(); } - FileUtils.deleteDirectory(CENTRAL_REPO_DIRECTORY_PATH.toFile()); - } catch (IOException | CentralRepoException ex) { + FileUtil.deleteDir(CENTRAL_REPO_DIRECTORY_PATH.toFile()); + } catch (CentralRepoException ex) { Exceptions.printStackTrace(ex); Assert.fail(ex.getMessage()); } diff --git a/Experimental/nbproject/project.xml b/Experimental/nbproject/project.xml index 1679c1c123..5f0b81a006 100644 --- a/Experimental/nbproject/project.xml +++ b/Experimental/nbproject/project.xml @@ -134,7 +134,8 @@ - 1.0 + 1 + 23 diff --git a/KeywordSearch/nbproject/project.xml b/KeywordSearch/nbproject/project.xml index 930acbf4db..bc85fdb6e5 100644 --- a/KeywordSearch/nbproject/project.xml +++ b/KeywordSearch/nbproject/project.xml @@ -118,7 +118,8 @@ - 1.0 + 1 + 23 diff --git a/Tika/manifest.mf b/Tika/manifest.mf index 770d374a5d..d5e404d1e3 100755 --- a/Tika/manifest.mf +++ b/Tika/manifest.mf @@ -1,6 +1,7 @@ Manifest-Version: 1.0 AutoUpdate-Show-In-Client: true -OpenIDE-Module: org.sleuthkit.autopsy.Tika +OpenIDE-Module: org.sleuthkit.autopsy.Tika/1 +OpenIDE-Module-Implementation-Version: 1 OpenIDE-Module-Localizing-Bundle: org/sleuthkit/autopsy/Tika/Bundle.properties -OpenIDE-Module-Specification-Version: 1.0 +OpenIDE-Module-Specification-Version: 23 diff --git a/ZookeeperNodeMigration/.gitignore b/ZookeeperNodeMigration/.gitignore new file mode 100755 index 0000000000..0e56a449c7 --- /dev/null +++ b/ZookeeperNodeMigration/.gitignore @@ -0,0 +1,3 @@ +/nbproject/private/ +/build/ + diff --git a/ZookeeperNodeMigration/build.xml b/ZookeeperNodeMigration/build.xml new file mode 100755 index 0000000000..9705ad3562 --- /dev/null +++ b/ZookeeperNodeMigration/build.xml @@ -0,0 +1,73 @@ + + + + + + + + + + + Builds, tests, and runs the project ZookeeperNodeMigration. + + + diff --git a/ZookeeperNodeMigration/dist/README.TXT b/ZookeeperNodeMigration/dist/README.TXT new file mode 100755 index 0000000000..087be91dc3 --- /dev/null +++ b/ZookeeperNodeMigration/dist/README.TXT @@ -0,0 +1,32 @@ +======================== +BUILD OUTPUT DESCRIPTION +======================== + +When you build an Java application project that has a main class, the IDE +automatically copies all of the JAR +files on the projects classpath to your projects dist/lib folder. The IDE +also adds each of the JAR files to the Class-Path element in the application +JAR files manifest file (MANIFEST.MF). + +To run the project from the command line, go to the dist folder and +type the following: + +java -jar "ZookeeperNodeMigration.jar" + +To distribute this project, zip up the dist folder (including the lib folder) +and distribute the ZIP file. + +Notes: + +* If two JAR files on the project classpath have the same name, only the first +JAR file is copied to the lib folder. +* Only JAR files are copied to the lib folder. +If the classpath contains other types of files or folders, these files (folders) +are not copied. +* If a library on the projects classpath also has a Class-Path element +specified in the manifest,the content of the Class-Path element has to be on +the projects runtime path. +* To set a main class in a standard Java project, right-click the project node +in the Projects window and choose Properties. Then click Run and enter the +class name in the Main Class field. Alternatively, you can manually type the +class name in the manifest Main-Class element. diff --git a/ZookeeperNodeMigration/dist/ZookeeperNodeMigration.jar b/ZookeeperNodeMigration/dist/ZookeeperNodeMigration.jar new file mode 100755 index 0000000000..92b11bc86a Binary files /dev/null and b/ZookeeperNodeMigration/dist/ZookeeperNodeMigration.jar differ diff --git a/ZookeeperNodeMigration/dist/lib/curator-client-2.8.0.jar b/ZookeeperNodeMigration/dist/lib/curator-client-2.8.0.jar new file mode 100755 index 0000000000..4ccc265cc4 Binary files /dev/null and b/ZookeeperNodeMigration/dist/lib/curator-client-2.8.0.jar differ diff --git a/ZookeeperNodeMigration/dist/lib/curator-framework-2.8.0.jar b/ZookeeperNodeMigration/dist/lib/curator-framework-2.8.0.jar new file mode 100755 index 0000000000..5e488892d1 Binary files /dev/null and b/ZookeeperNodeMigration/dist/lib/curator-framework-2.8.0.jar differ diff --git a/ZookeeperNodeMigration/dist/lib/curator-recipes-2.8.0.jar b/ZookeeperNodeMigration/dist/lib/curator-recipes-2.8.0.jar new file mode 100755 index 0000000000..34eb9c9677 Binary files /dev/null and b/ZookeeperNodeMigration/dist/lib/curator-recipes-2.8.0.jar differ diff --git a/ZookeeperNodeMigration/dist/lib/guava-17.0.jar b/ZookeeperNodeMigration/dist/lib/guava-17.0.jar new file mode 100755 index 0000000000..661fc7473f Binary files /dev/null and b/ZookeeperNodeMigration/dist/lib/guava-17.0.jar differ diff --git a/ZookeeperNodeMigration/dist/lib/log4j-1.2.17.jar b/ZookeeperNodeMigration/dist/lib/log4j-1.2.17.jar new file mode 100755 index 0000000000..1d425cf7d7 Binary files /dev/null and b/ZookeeperNodeMigration/dist/lib/log4j-1.2.17.jar differ diff --git a/ZookeeperNodeMigration/dist/lib/slf4j-api-1.7.24.jar b/ZookeeperNodeMigration/dist/lib/slf4j-api-1.7.24.jar new file mode 100755 index 0000000000..05941a12f0 Binary files /dev/null and b/ZookeeperNodeMigration/dist/lib/slf4j-api-1.7.24.jar differ diff --git a/ZookeeperNodeMigration/dist/lib/slf4j-log4j12-1.7.6.jar b/ZookeeperNodeMigration/dist/lib/slf4j-log4j12-1.7.6.jar new file mode 100755 index 0000000000..d1cc2456e4 Binary files /dev/null and b/ZookeeperNodeMigration/dist/lib/slf4j-log4j12-1.7.6.jar differ diff --git a/ZookeeperNodeMigration/dist/lib/zookeeper-3.4.6.jar b/ZookeeperNodeMigration/dist/lib/zookeeper-3.4.6.jar new file mode 100755 index 0000000000..7c340be9f5 Binary files /dev/null and b/ZookeeperNodeMigration/dist/lib/zookeeper-3.4.6.jar differ diff --git a/ZookeeperNodeMigration/docs/README.TXT b/ZookeeperNodeMigration/docs/README.TXT new file mode 100755 index 0000000000..9f5b8f4ca9 --- /dev/null +++ b/ZookeeperNodeMigration/docs/README.TXT @@ -0,0 +1,23 @@ +To run the project from the command line, go to the folder that contains "ZookeeperNodeMigration.jar" and +type the following: + +java -jar ZookeeperNodeMigration.jar + +To distribute this project, zip up the dist folder (including the lib folder) +and distribute the ZIP file. + +Usage: +ZookeeperNodeMigration input needs to be: +[Input Zookeeper IP Address or Hostname] [Input Zookeeper Port Number] [Output Zookeeper IP Address or Hostname] [Output Zookeeper Port Number] + +For example, if you execute the following command from command line line, the Zookeeper +nodes will get copied from Zookeeper server on localhost:9983 to Zookeeper server on localhost:19983 : + +java -jar ZookeeperNodeMigration.jar localhost 9983 localhost 19983 + + +If you do not have Java installed on the machine, you can use the packaged version of Java that is distributed along +with Autopsy. For example: + +"C:\Program Files\Autopsy-4.16.0\jre\bin\java.exe" -jar ZookeeperNodeMigration.jar localhost 9983 localhost 19983 + diff --git a/ZookeeperNodeMigration/manifest.mf b/ZookeeperNodeMigration/manifest.mf new file mode 100755 index 0000000000..328e8e5bc3 --- /dev/null +++ b/ZookeeperNodeMigration/manifest.mf @@ -0,0 +1,3 @@ +Manifest-Version: 1.0 +X-COMMENT: Main-Class will be added automatically by build + diff --git a/ZookeeperNodeMigration/nbproject/build-impl.xml b/ZookeeperNodeMigration/nbproject/build-impl.xml new file mode 100755 index 0000000000..76af3c9f63 --- /dev/null +++ b/ZookeeperNodeMigration/nbproject/build-impl.xml @@ -0,0 +1,1770 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Must set src.dir + Must set test.src.dir + Must set build.dir + Must set dist.dir + Must set build.classes.dir + Must set dist.javadoc.dir + Must set build.test.classes.dir + Must set build.test.results.dir + Must set build.classes.excludes + Must set dist.jar + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Must set javac.includes + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + No tests executed. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Must set JVM to use for profiling in profiler.info.jvm + Must set profiler agent JVM arguments in profiler.info.jvmargs.agent + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Must select some files in the IDE or set javac.includes + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + To run this application from the command line without Ant, try: + + java -jar "${dist.jar.resolved}" + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Must select one file in the IDE or set run.class + + + + Must select one file in the IDE or set run.class + + + + + + + + + + + + + + + + + + + + + + + Must select one file in the IDE or set debug.class + + + + + Must select one file in the IDE or set debug.class + + + + + Must set fix.includes + + + + + + + + + + This target only works when run from inside the NetBeans IDE. + + + + + + + + + Must select one file in the IDE or set profile.class + This target only works when run from inside the NetBeans IDE. + + + + + + + + + This target only works when run from inside the NetBeans IDE. + + + + + + + + + + + + + This target only works when run from inside the NetBeans IDE. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Must select one file in the IDE or set run.class + + + + + + Must select some files in the IDE or set test.includes + + + + + Must select one file in the IDE or set run.class + + + + + Must select one file in the IDE or set applet.url + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Must select some files in the IDE or set javac.includes + + + + + + + + + + + + + + + + + + + + + + + + Some tests failed; see details above. + + + + + + + + + Must select some files in the IDE or set test.includes + + + + Some tests failed; see details above. + + + + Must select some files in the IDE or set test.class + Must select some method in the IDE or set test.method + + + + Some tests failed; see details above. + + + + + Must select one file in the IDE or set test.class + + + + Must select one file in the IDE or set test.class + Must select some method in the IDE or set test.method + + + + + + + + + + + + + + Must select one file in the IDE or set applet.url + + + + + + + + + Must select one file in the IDE or set applet.url + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/ZookeeperNodeMigration/nbproject/project.properties b/ZookeeperNodeMigration/nbproject/project.properties new file mode 100755 index 0000000000..26892390a5 --- /dev/null +++ b/ZookeeperNodeMigration/nbproject/project.properties @@ -0,0 +1,107 @@ +annotation.processing.enabled=true +annotation.processing.enabled.in.editor=false +annotation.processing.processors.list= +annotation.processing.run.all.processors=true +annotation.processing.source.output=${build.generated.sources.dir}/ap-source-output +application.title=ZookeeperNodeMigration +application.vendor=elivis +build.classes.dir=${build.dir}/classes +build.classes.excludes=**/*.java,**/*.form +# This directory is removed when the project is cleaned: +build.dir=build +build.generated.dir=${build.dir}/generated +build.generated.sources.dir=${build.dir}/generated-sources +# Only compile against the classpath explicitly listed here: +build.sysclasspath=ignore +build.test.classes.dir=${build.dir}/test/classes +build.test.results.dir=${build.dir}/test/results +# Uncomment to specify the preferred debugger connection transport: +#debug.transport=dt_socket +debug.classpath=\ + ${run.classpath} +debug.modulepath=\ + ${run.modulepath} +debug.test.classpath=\ + ${run.test.classpath} +debug.test.modulepath=\ + ${run.test.modulepath} +# Files in build.classes.dir which should be excluded from distribution jar +dist.archive.excludes= +# This directory is removed when the project is cleaned: +dist.dir=dist +dist.jar=${dist.dir}/ZookeeperNodeMigration.jar +dist.javadoc.dir=${dist.dir}/javadoc +endorsed.classpath= +excludes= +file.reference.curator-client-2.8.0.jar=release/curator-client-2.8.0.jar +file.reference.curator-framework-2.8.0.jar=release/curator-framework-2.8.0.jar +file.reference.curator-recipes-2.8.0.jar=release/curator-recipes-2.8.0.jar +file.reference.guava-17.0.jar=release/guava-17.0.jar +file.reference.log4j-1.2.17.jar=release/log4j-1.2.17.jar +file.reference.slf4j-api-1.7.24.jar=release/slf4j-api-1.7.24.jar +file.reference.slf4j-log4j12-1.7.6.jar=release/slf4j-log4j12-1.7.6.jar +file.reference.zookeeper-3.4.6.jar=release/zookeeper-3.4.6.jar +includes=** +jar.compress=false +javac.classpath=\ + ${file.reference.curator-client-2.8.0.jar}:\ + ${file.reference.curator-framework-2.8.0.jar}:\ + ${file.reference.curator-recipes-2.8.0.jar}:\ + ${file.reference.zookeeper-3.4.6.jar}:\ + ${file.reference.slf4j-api-1.7.24.jar}:\ + ${file.reference.slf4j-log4j12-1.7.6.jar}:\ + ${file.reference.log4j-1.2.17.jar}:\ + ${file.reference.guava-17.0.jar} +# Space-separated list of extra javac options +javac.compilerargs= +javac.deprecation=false +javac.external.vm=true +javac.modulepath= +javac.processormodulepath= +javac.processorpath=\ + ${javac.classpath} +javac.source=1.8 +javac.target=1.8 +javac.test.classpath=\ + ${javac.classpath}:\ + ${build.classes.dir} +javac.test.modulepath=\ + ${javac.modulepath} +javac.test.processorpath=\ + ${javac.test.classpath} +javadoc.additionalparam= +javadoc.author=false +javadoc.encoding=${source.encoding} +javadoc.html5=false +javadoc.noindex=false +javadoc.nonavbar=false +javadoc.notree=false +javadoc.private=false +javadoc.splitindex=true +javadoc.use=true +javadoc.version=false +javadoc.windowtitle= +jlink.launcher=false +jlink.launcher.name=ZookeeperNodeMigration +main.class=zookeepernodemigration.ZookeeperNodeMigration +manifest.file=manifest.mf +meta.inf.dir=${src.dir}/META-INF +mkdist.disabled=false +platform.active=default_platform +run.classpath=\ + ${javac.classpath}:\ + ${build.classes.dir} +# Space-separated list of JVM arguments used when running the project. +# You may also define separate properties like run-sys-prop.name=value instead of -Dname=value. +# To set system properties for unit tests define test-sys-prop.name=value: +run.jvmargs= +run.modulepath=\ + ${javac.modulepath} +run.test.classpath=\ + ${javac.test.classpath}:\ + ${build.test.classes.dir} +run.test.modulepath=\ + ${javac.test.modulepath} +source.encoding=UTF-8 +src.dir=src +test.src.dir=test diff --git a/ZookeeperNodeMigration/nbproject/project.xml b/ZookeeperNodeMigration/nbproject/project.xml new file mode 100755 index 0000000000..1db04676a2 --- /dev/null +++ b/ZookeeperNodeMigration/nbproject/project.xml @@ -0,0 +1,15 @@ + + + org.netbeans.modules.java.j2seproject + + + ZookeeperNodeMigration + + + + + + + + + diff --git a/ZookeeperNodeMigration/release/curator-client-2.8.0.jar b/ZookeeperNodeMigration/release/curator-client-2.8.0.jar new file mode 100755 index 0000000000..4ccc265cc4 Binary files /dev/null and b/ZookeeperNodeMigration/release/curator-client-2.8.0.jar differ diff --git a/ZookeeperNodeMigration/release/curator-framework-2.8.0.jar b/ZookeeperNodeMigration/release/curator-framework-2.8.0.jar new file mode 100755 index 0000000000..5e488892d1 Binary files /dev/null and b/ZookeeperNodeMigration/release/curator-framework-2.8.0.jar differ diff --git a/ZookeeperNodeMigration/release/curator-recipes-2.8.0.jar b/ZookeeperNodeMigration/release/curator-recipes-2.8.0.jar new file mode 100755 index 0000000000..34eb9c9677 Binary files /dev/null and b/ZookeeperNodeMigration/release/curator-recipes-2.8.0.jar differ diff --git a/ZookeeperNodeMigration/release/guava-17.0.jar b/ZookeeperNodeMigration/release/guava-17.0.jar new file mode 100755 index 0000000000..661fc7473f Binary files /dev/null and b/ZookeeperNodeMigration/release/guava-17.0.jar differ diff --git a/ZookeeperNodeMigration/release/log4j-1.2.17.jar b/ZookeeperNodeMigration/release/log4j-1.2.17.jar new file mode 100755 index 0000000000..1d425cf7d7 Binary files /dev/null and b/ZookeeperNodeMigration/release/log4j-1.2.17.jar differ diff --git a/ZookeeperNodeMigration/release/slf4j-api-1.7.24.jar b/ZookeeperNodeMigration/release/slf4j-api-1.7.24.jar new file mode 100755 index 0000000000..05941a12f0 Binary files /dev/null and b/ZookeeperNodeMigration/release/slf4j-api-1.7.24.jar differ diff --git a/ZookeeperNodeMigration/release/slf4j-log4j12-1.7.6.jar b/ZookeeperNodeMigration/release/slf4j-log4j12-1.7.6.jar new file mode 100755 index 0000000000..d1cc2456e4 Binary files /dev/null and b/ZookeeperNodeMigration/release/slf4j-log4j12-1.7.6.jar differ diff --git a/ZookeeperNodeMigration/release/zookeeper-3.4.6.jar b/ZookeeperNodeMigration/release/zookeeper-3.4.6.jar new file mode 100755 index 0000000000..7c340be9f5 Binary files /dev/null and b/ZookeeperNodeMigration/release/zookeeper-3.4.6.jar differ diff --git a/ZookeeperNodeMigration/src/zookeepernodemigration/AutoIngestJobNodeData.java b/ZookeeperNodeMigration/src/zookeepernodemigration/AutoIngestJobNodeData.java new file mode 100755 index 0000000000..82e75a7b1c --- /dev/null +++ b/ZookeeperNodeMigration/src/zookeepernodemigration/AutoIngestJobNodeData.java @@ -0,0 +1,687 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2020 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package zookeepernodemigration; + +import java.io.Serializable; +import java.nio.BufferUnderflowException; +import java.nio.ByteBuffer; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Date; +import javax.lang.model.type.TypeKind; + +/** + * An object that converts auto ingest job data for an auto ingest job + * coordination service node to and from byte arrays. + */ +final class AutoIngestJobNodeData { + + private static final int CURRENT_VERSION = 2; + private static final int DEFAULT_PRIORITY = 0; + + /* + * This number is the sum of each piece of data, based on it's type. For the + * types boolean, int, and long, values 1, 4, and 8 will be added + * respectively. For String objects, the length of the string, plus either a + * byte or short respesenting the length of the string, will be added. + * + * This field is used to set the size of the buffer during the byte array + * creation in the 'toArray()' method. Since the final size of the array + * isn't immediately known at the time of creation, this number is used to + * create an array as large as could possibly be needed to store all the + * data. This avoids the need to continuously enlarge the buffer. Once the + * buffer has all the necessary data, it will be resized as appropriate. + */ + private static final int MAX_POSSIBLE_NODE_DATA_SIZE = 131637; + + /* + * Version 0 fields. + */ + private int processingStatus; + private int priority; + private int numberOfCrashes; + private long completedDate; + private boolean errorsOccurred; + + /* + * Version 1 fields. + */ + private int version; + private String manifestFilePath; // 'short' length used in byte array + private long manifestFileDate; + private String caseName; // 'byte' length used in byte array + private String deviceId; // 'byte' length used in byte array + private String dataSourcePath; // 'short' length used in byte array + private String caseDirectoryPath; // 'short' length used in byte array + private String processingHostName; // 'short' length used in byte array + private byte processingStage; + private long processingStageStartDate; + private String processingStageDetailsDescription; // 'byte' length used in byte array + private long processingStageDetailsStartDate; + + /* + * Version 2 fields. + */ + private long dataSourceSize; + + /** + * Processing statuses for an auto ingest job. + */ + enum ProcessingStatus { + PENDING, + PROCESSING, + COMPLETED, + DELETED + } + + /** + * Processing stages for an auto ingest job. + */ + enum Stage { + + PENDING("Pending"), + STARTING("Starting"), + UPDATING_SHARED_CONFIG("Updating shared configuration"), + CHECKING_SERVICES("Checking services"), + OPENING_CASE("Opening case"), + IDENTIFYING_DATA_SOURCE("Identifying data source type"), + ADDING_DATA_SOURCE("Adding data source"), + ANALYZING_DATA_SOURCE("Analyzing data source"), + ANALYZING_FILES("Analyzing files"), + EXPORTING_FILES("Exporting files"), + CANCELLING_MODULE("Cancelling module"), + CANCELLING("Cancelling"), + COMPLETED("Completed"); + + private final String displayText; + + private Stage(String displayText) { + this.displayText = displayText; + } + + String getDisplayText() { + return displayText; + } + + } + + + /** + * Processing stage details for an auto ingest job. + */ + static final class StageDetails implements Serializable { + + private static final long serialVersionUID = 1L; + private final String description; + private final Date startDate; + + StageDetails(String description, Date startDate) { + this.description = description; + this.startDate = startDate; + } + + String getDescription() { + return this.description; + } + + Date getStartDate() { + return new Date(this.startDate.getTime()); + } + + } + + /** + * Gets the current version of the auto ingest job coordination service node + * data. + * + * @return The version number. + */ + static int getCurrentVersion() { + return AutoIngestJobNodeData.CURRENT_VERSION; + } + + /** + * Uses a coordination service node data to construct an object that + * converts auto ingest job data for an auto ingest job coordination service + * node to and from byte arrays. + * + * @param nodeData The raw bytes received from the coordination service. + */ + AutoIngestJobNodeData(byte[] nodeData) throws InvalidDataException { + if (null == nodeData || nodeData.length == 0) { + throw new InvalidDataException(null == nodeData ? "Null nodeData byte array" : "Zero-length nodeData byte array"); + } + + /* + * Set default values for all fields. + */ + this.processingStatus = ProcessingStatus.PENDING.ordinal(); + this.priority = DEFAULT_PRIORITY; + this.numberOfCrashes = 0; + this.completedDate = 0L; + this.errorsOccurred = false; + this.version = 0; + this.manifestFilePath = ""; + this.manifestFileDate = 0L; + this.caseName = ""; + this.deviceId = ""; + this.dataSourcePath = ""; + this.caseDirectoryPath = ""; + this.processingHostName = ""; + this.processingStage = (byte) Stage.PENDING.ordinal(); + this.processingStageStartDate = 0L; + this.processingStageDetailsDescription = ""; + this.processingStageDetailsStartDate = 0L; + this.dataSourceSize = 0L; + + /* + * Get fields from node data. + */ + ByteBuffer buffer = ByteBuffer.wrap(nodeData); + try { + if (buffer.hasRemaining()) { + /* + * Get version 0 fields. + */ + this.processingStatus = buffer.getInt(); + this.priority = buffer.getInt(); + this.numberOfCrashes = buffer.getInt(); + this.completedDate = buffer.getLong(); + int errorFlag = buffer.getInt(); + this.errorsOccurred = (1 == errorFlag); + } + + if (buffer.hasRemaining()) { + /* + * Get version 1 fields. + */ + this.version = buffer.getInt(); + this.deviceId = getStringFromBuffer(buffer, TypeKind.BYTE); + this.caseName = getStringFromBuffer(buffer, TypeKind.BYTE); + this.caseDirectoryPath = getStringFromBuffer(buffer, TypeKind.SHORT); + this.manifestFileDate = buffer.getLong(); + this.manifestFilePath = getStringFromBuffer(buffer, TypeKind.SHORT); + this.dataSourcePath = getStringFromBuffer(buffer, TypeKind.SHORT); + this.processingStage = buffer.get(); + this.processingStageStartDate = buffer.getLong(); + this.processingStageDetailsDescription = getStringFromBuffer(buffer, TypeKind.BYTE); + this.processingStageDetailsStartDate = buffer.getLong(); + this.processingHostName = getStringFromBuffer(buffer, TypeKind.SHORT); + } + + if (buffer.hasRemaining()) { + /* + * Get version 2 fields. + */ + this.dataSourceSize = buffer.getLong(); + } + + } catch (BufferUnderflowException ex) { + throw new InvalidDataException("Node data is incomplete", ex); + } + } + + /** + * Gets the processing status of the job. + * + * @return The processing status. + */ + ProcessingStatus getProcessingStatus() { + return ProcessingStatus.values()[this.processingStatus]; + } + + /** + * Sets the processing status of the job. + * + * @param processingSatus The processing status. + */ + void setProcessingStatus(ProcessingStatus processingStatus) { + this.processingStatus = processingStatus.ordinal(); + } + + /** + * Gets the priority of the job. + * + * @return The priority. + */ + int getPriority() { + return this.priority; + } + + /** + * Sets the priority of the job. A higher number indicates a higheer + * priority. + * + * @param priority The priority. + */ + void setPriority(int priority) { + this.priority = priority; + } + + /** + * Gets the number of times the job has crashed during processing. + * + * @return The number of crashes. + */ + int getNumberOfCrashes() { + return this.numberOfCrashes; + } + + /** + * Sets the number of times the job has crashed during processing. + * + * @param numberOfCrashes The number of crashes. + */ + void setNumberOfCrashes(int numberOfCrashes) { + this.numberOfCrashes = numberOfCrashes; + } + + /** + * Gets the date the job was completed. A completion date equal to the epoch + * (January 1, 1970, 00:00:00 GMT), i.e., Date.getTime() returns 0L, + * indicates the job has not been completed. + * + * @return The job completion date. + */ + Date getCompletedDate() { + return new Date(this.completedDate); + } + + /** + * Sets the date the job was completed. A completion date equal to the epoch + * (January 1, 1970, 00:00:00 GMT), i.e., Date.getTime() returns 0L, + * indicates the job has not been completed. + * + * @param completedDate The job completion date. + */ + void setCompletedDate(Date completedDate) { + this.completedDate = completedDate.getTime(); + } + + /** + * Gets whether or not any errors occurred during the processing of the job. + * + * @return True or false. + */ + boolean getErrorsOccurred() { + return this.errorsOccurred; + } + + /** + * Sets whether or not any errors occurred during the processing of job. + * + * @param errorsOccurred True or false. + */ + void setErrorsOccurred(boolean errorsOccurred) { + this.errorsOccurred = errorsOccurred; + } + + /** + * Gets the node data version number. + * + * @return The version number. + */ + int getVersion() { + return this.version; + } + + /** + * Gets the device ID of the device associated with the data source for the + * job. + * + * @return The device ID. + */ + String getDeviceId() { + return this.deviceId; + } + + /** + * Sets the device ID of the device associated with the data source for the + * job. + * + * @param deviceId The device ID. + */ + void setDeviceId(String deviceId) { + this.deviceId = deviceId; + } + + /** + * Gets the case name. + * + * @return The case name. + */ + String getCaseName() { + return this.caseName; + } + + /** + * Sets the case name. + * + * @param caseName The case name. + */ + void setCaseName(String caseName) { + this.caseName = caseName; + } + + /** + * Sets the path to the case directory of the case associated with the job. + * + * @param caseDirectoryPath The path to the case directory. + */ + synchronized void setCaseDirectoryPath(Path caseDirectoryPath) { + if (caseDirectoryPath == null) { + this.caseDirectoryPath = ""; + } else { + this.caseDirectoryPath = caseDirectoryPath.toString(); + } + } + + /** + * Gets the path to the case directory of the case associated with the job. + * + * @return The case directory path or an empty string path if the case + * directory has not been created yet. + */ + synchronized Path getCaseDirectoryPath() { + if (!caseDirectoryPath.isEmpty()) { + return Paths.get(caseDirectoryPath); + } else { + return Paths.get(""); + } + } + + /** + * Gets the date the manifest was created. + * + * @return The date the manifest was created. + */ + Date getManifestFileDate() { + return new Date(this.manifestFileDate); + } + + /** + * Sets the date the manifest was created. + * + * @param manifestFileDate The date the manifest was created. + */ + void setManifestFileDate(Date manifestFileDate) { + this.manifestFileDate = manifestFileDate.getTime(); + } + + /** + * Gets the manifest file path. + * + * @return The manifest file path. + */ + Path getManifestFilePath() { + return Paths.get(this.manifestFilePath); + } + + /** + * Sets the manifest file path. + * + * @param manifestFilePath The manifest file path. + */ + void setManifestFilePath(Path manifestFilePath) { + if (manifestFilePath != null) { + this.manifestFilePath = manifestFilePath.toString(); + } else { + this.manifestFilePath = ""; + } + } + + /** + * Gets the path of the data source for the job. + * + * @return The data source path. + */ + Path getDataSourcePath() { + return Paths.get(dataSourcePath); + } + + /** + * Get the file name portion of the path of the data source for the job. + * + * @return The data source file name. + */ + public String getDataSourceFileName() { + return Paths.get(dataSourcePath).getFileName().toString(); + } + + /** + * Sets the path of the data source for the job. + * + * @param dataSourcePath The data source path. + */ + void setDataSourcePath(Path dataSourcePath) { + if (dataSourcePath != null) { + this.dataSourcePath = dataSourcePath.toString(); + } else { + this.dataSourcePath = ""; + } + } + + /** + * Get the processing stage of the job. + * + * @return The processing stage. + */ + Stage getProcessingStage() { + return Stage.values()[this.processingStage]; + } + + /** + * Sets the processing stage job. + * + * @param processingStage The processing stage. + */ + void setProcessingStage(Stage processingStage) { + this.processingStage = (byte) processingStage.ordinal(); + } + + /** + * Gets the processing stage start date. + * + * @return The processing stage start date. + */ + Date getProcessingStageStartDate() { + return new Date(this.processingStageStartDate); + } + + /** + * Sets the processing stage start date. + * + * @param processingStageStartDate The processing stage start date. + */ + void setProcessingStageStartDate(Date processingStageStartDate) { + this.processingStageStartDate = processingStageStartDate.getTime(); + } + + /** + * Get the processing stage details. + * + * @return A processing stage details object. + */ + StageDetails getProcessingStageDetails() { + return new StageDetails(this.processingStageDetailsDescription, new Date(this.processingStageDetailsStartDate)); + } + + /** + * Sets the details of the current processing stage. + * + * @param stageDetails A stage details object. + */ + void setProcessingStageDetails(StageDetails stageDetails) { + this.processingStageDetailsDescription = stageDetails.getDescription(); + this.processingStageDetailsStartDate = stageDetails.getStartDate().getTime(); + } + + /** + * Gets the processing host name, may be the empty string. + * + * @return The processing host. The empty string if the job is not currently + * being processed. + */ + String getProcessingHostName() { + return this.processingHostName; + } + + /** + * Sets the processing host name. May be the empty string. + * + * @param processingHost The processing host name. The empty string if the + * job is not currently being processed. + */ + void setProcessingHostName(String processingHost) { + this.processingHostName = processingHost; + } + + /** + * Gets the total size of the data source. + * + * @return The data source size. + */ + long getDataSourceSize() { + return this.dataSourceSize; + } + + /** + * Sets the total size of the data source. + * + * @param dataSourceSize The data source size. + */ + void setDataSourceSize(long dataSourceSize) { + this.dataSourceSize = dataSourceSize; + } + + /** + * Gets the node data as a byte array that can be sent to the coordination + * service. + * + * @return The node data as a byte array. + */ + byte[] toArray() { + ByteBuffer buffer = ByteBuffer.allocate(MAX_POSSIBLE_NODE_DATA_SIZE); + + // Write data (compatible with version 0) + buffer.putInt(this.processingStatus); + buffer.putInt(this.priority); + buffer.putInt(this.numberOfCrashes); + buffer.putLong(this.completedDate); + buffer.putInt(this.errorsOccurred ? 1 : 0); + + if (this.version >= 1) { + // Write version + buffer.putInt(this.version); + + // Write data + putStringIntoBuffer(deviceId, buffer, TypeKind.BYTE); + putStringIntoBuffer(caseName, buffer, TypeKind.BYTE); + putStringIntoBuffer(caseDirectoryPath, buffer, TypeKind.SHORT); + buffer.putLong(this.manifestFileDate); + putStringIntoBuffer(manifestFilePath, buffer, TypeKind.SHORT); + putStringIntoBuffer(dataSourcePath, buffer, TypeKind.SHORT); + buffer.put(this.processingStage); + buffer.putLong(this.processingStageStartDate); + putStringIntoBuffer(this.processingStageDetailsDescription, buffer, TypeKind.BYTE); + buffer.putLong(this.processingStageDetailsStartDate); + putStringIntoBuffer(processingHostName, buffer, TypeKind.SHORT); + + if (this.version >= 2) { + buffer.putLong(this.dataSourceSize); + } + } + + // Prepare the array + byte[] array = new byte[buffer.position()]; + buffer.rewind(); + buffer.get(array, 0, array.length); + + return array; + } + + /** + * This method retrieves a string from a given buffer. Depending on the type + * specified, either a 'byte' or a 'short' will first be read out of the + * buffer which gives the length of the string so it can be properly parsed. + * + * @param buffer The buffer from which the string will be read. + * @param lengthType The size of the length data. + * + * @return The string read from the buffer. + */ + private String getStringFromBuffer(ByteBuffer buffer, TypeKind lengthType) { + int length = 0; + String output = ""; + + switch (lengthType) { + case BYTE: + length = buffer.get(); + break; + case SHORT: + length = buffer.getShort(); + break; + } + + if (length > 0) { + byte[] array = new byte[length]; + buffer.get(array, 0, length); + output = new String(array); + } + + return output; + } + + /** + * This method puts a given string into a given buffer. Depending on the + * type specified, either a 'byte' or a 'short' will be inserted prior to + * the string which gives the length of the string so it can be properly + * parsed. + * + * @param stringValue The string to write to the buffer. + * @param buffer The buffer to which the string will be written. + * @param lengthType The size of the length data. + */ + private void putStringIntoBuffer(String stringValue, ByteBuffer buffer, TypeKind lengthType) { + switch (lengthType) { + case BYTE: + buffer.put((byte) stringValue.length()); + break; + case SHORT: + buffer.putShort((short) stringValue.length()); + break; + } + + buffer.put(stringValue.getBytes()); + } + + final static class InvalidDataException extends Exception { + + private static final long serialVersionUID = 1L; + + private InvalidDataException(String message) { + super(message); + } + + private InvalidDataException(String message, Throwable cause) { + super(message, cause); + } + } + +} diff --git a/ZookeeperNodeMigration/src/zookeepernodemigration/TimeStampUtils.java b/ZookeeperNodeMigration/src/zookeepernodemigration/TimeStampUtils.java new file mode 100755 index 0000000000..c04feca3c1 --- /dev/null +++ b/ZookeeperNodeMigration/src/zookeepernodemigration/TimeStampUtils.java @@ -0,0 +1,93 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2020 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package zookeepernodemigration; + +import java.text.SimpleDateFormat; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Utility methods for working with time stamps of the form + * 'yyyy_MM_dd_HH_mm_ss'. + */ +final class TimeStampUtils { + + /* + * Sample time stamp suffix: 2015_02_02_12_10_31 + */ + private static final Pattern TIME_STAMP_PATTERN = Pattern.compile("\\d{4}_\\d{2}_\\d{2}_\\d{2}_\\d{2}_\\d{2}$"); + private static final int LENGTH_OF_DATE_TIME_STAMP = 20; // length of the above time stamp + + /** + * Checks whether a string ends with a time stamp. + * + * @param inputString The string to check. + * + * @return True or false. + */ + static boolean endsWithTimeStamp(String inputString) { + Matcher m = TIME_STAMP_PATTERN.matcher(inputString); + return m.find(); + } + + /** + * Gets the fixed length of the time stamp suffix. + * + * @return The length. + */ + static int getTimeStampLength() { + return LENGTH_OF_DATE_TIME_STAMP; + } + + /** + * Removes the time stamp suffix from a string, if present. + * + * @param inputString The string to trim. + * + * @return The trimmed string. + */ + static String removeTimeStamp(String inputString) { + String trimmedString = inputString; + if (inputString != null && endsWithTimeStamp(inputString)) { + trimmedString = inputString.substring(0, inputString.length() - getTimeStampLength()); + } + return trimmedString; + } + + /** + * Gets the time stamp suffix from a string, if present. + * + * @param inputString the name to check for a timestamp + * + * @return The time stamp, may be the empty. + */ + static String getTimeStampOnly(String inputString) { + String timeStamp = ""; + if (inputString != null && endsWithTimeStamp(inputString)) { + timeStamp = inputString.substring(inputString.length() - getTimeStampLength(), inputString.length()); + } + return timeStamp; + } + + /* + * Private contructor to prevent instantiation. + */ + private TimeStampUtils() { + } +} diff --git a/ZookeeperNodeMigration/src/zookeepernodemigration/ZookeeperNodeMigration.java b/ZookeeperNodeMigration/src/zookeepernodemigration/ZookeeperNodeMigration.java new file mode 100755 index 0000000000..c62db26b84 --- /dev/null +++ b/ZookeeperNodeMigration/src/zookeepernodemigration/ZookeeperNodeMigration.java @@ -0,0 +1,464 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2020 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package zookeepernodemigration; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.logging.Level; +import java.util.logging.Logger; +import org.apache.curator.RetryPolicy; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.CuratorFrameworkFactory; +import org.apache.curator.framework.recipes.locks.InterProcessMutex; +import org.apache.curator.framework.recipes.locks.InterProcessReadWriteLock; +import org.apache.curator.retry.ExponentialBackoffRetry; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.KeeperException.NoNodeException; +import org.apache.zookeeper.WatchedEvent; +import org.apache.zookeeper.ZooDefs; +import org.apache.zookeeper.ZooKeeper; + +/** + * Utility to migrate Autopsy coordination service data from one ZK database to + * another. + */ +public class ZookeeperNodeMigration { + + private static final Logger LOGGER = Logger.getLogger(ZookeeperNodeMigration.class.getName()); + private static final int SESSION_TIMEOUT_MILLISECONDS = 300000; + private static final int CONNECTION_TIMEOUT_MILLISECONDS = 300000; + private static final int ZOOKEEPER_SESSION_TIMEOUT_MILLIS = 3000; + private static final int ZOOKEEPER_CONNECTION_TIMEOUT_MILLIS = 15000; + private static final String DEFAULT_NAMESPACE_ROOT = "autopsy"; + private static CuratorFramework inputCurator; + private static CuratorFramework outputCurator; + private static Map categoryNodeToPath; + + private ZookeeperNodeMigration(){ + } + + /** + * Main method. + * + * @param args the command line arguments + */ + public static void main(String[] args) { + + String inputZkIpAddr, inputZkPort, outputZkIpAddr, outputZkPort; + + if (args.length == 4) { + inputZkIpAddr = args[0]; + inputZkPort = args[1]; + outputZkIpAddr = args[2]; + outputZkPort = args[3]; + } else { + System.out.println("Input needs to be [Input Zookeeper IP Address] [Input Zookeeper Port Number] [Output Zookeeper IP Address] [Output Zookeeper Port Number]"); + LOGGER.log(Level.SEVERE, "Input needs to be [Input Zookeeper IP Address] [Input Zookeeper Port Number] [Output Zookeeper IP Address] [Output Zookeeper Port Number]"); + return; + } + + if (inputZkIpAddr.isEmpty() || inputZkPort.isEmpty() || outputZkIpAddr.isEmpty() || outputZkPort.isEmpty()) { + System.out.println("Input needs to be [Input Zookeeper IP Address] [Input Zookeeper Port Number] [Output Zookeeper IP Address] [Output Zookeeper Port Number]"); + LOGGER.log(Level.SEVERE, "Input needs to be [Input Zookeeper IP Address] [Input Zookeeper Port Number] [Output Zookeeper IP Address] [Output Zookeeper Port Number]"); + return; + } + + inputCurator = initializeCurator(inputZkIpAddr, inputZkPort); + if (inputCurator == null) { + System.out.println("Unable to initialize Zookeeper or Curator: " + inputZkIpAddr + ":" + inputZkPort); + LOGGER.log(Level.SEVERE, "Unable to initialize Zookeeper or Curator: {0}:{1}", new Object[]{inputZkIpAddr, inputZkPort}); + return; + } + + try { + categoryNodeToPath = populateCategoryNodes(inputCurator); + } catch (KeeperException | CoordinationServiceException ex) { + System.out.println("Unable to initialize Curator: " + inputZkIpAddr + ":" + inputZkPort); + LOGGER.log(Level.SEVERE, "Unable to initialize Curator: {0}:{1}", new Object[]{inputZkIpAddr, inputZkPort}); + return; + } + + outputCurator = initializeCurator(outputZkIpAddr, outputZkPort); + if (outputCurator == null) { + System.out.println("Unable to initialize Zookeeper or Curator: " + outputZkIpAddr + ":" + outputZkPort); + LOGGER.log(Level.SEVERE, "Unable to initialize Zookeeper or Curator: {0}:{1}", new Object[]{outputZkIpAddr, outputZkPort}); + return; + } + + try { + // if output ZK database is new, we may have to ceate root "autopsy" node and it's sub-nodes + populateCategoryNodes(outputCurator); + } catch (KeeperException | CoordinationServiceException ex) { + System.out.println("Unable to initialize Curator: " + outputZkIpAddr + ":" + outputZkPort); + LOGGER.log(Level.SEVERE, "Unable to initialize Curator: {0}:{1}", new Object[]{outputZkIpAddr, outputZkPort}); + return; + } + + copyAllCategoryNodes(); + + System.out.println("Done..."); + } + + /** + * Copy all Autopsy coordination service nodes from one ZK database to + * another. + */ + private static void copyAllCategoryNodes() { + + for (CategoryNode category : CategoryNode.values()) { + List inputNodeList = Collections.EMPTY_LIST; + try { + inputNodeList = getNodeList(category); + } catch (CoordinationServiceException ex) { + System.out.println("Unable to get ZK nodes for category: " + category.getDisplayName()); + LOGGER.log(Level.SEVERE, "Unable to get ZK nodes for category: " + category.getDisplayName(), ex); + continue; + } + + for (String zkNode : inputNodeList) { + try { + final byte[] nodeBytes = getNodeData(category, zkNode); + try (Lock manifestLock = tryGetExclusiveLock(outputCurator, category, zkNode)) { + setNodeData(outputCurator, category, zkNode, nodeBytes); + } + } catch (CoordinationServiceException | InterruptedException ex) { + System.out.println("Unable to write ZK node data for node: " + zkNode); + LOGGER.log(Level.SEVERE, "Unable to write ZK node data for node: " + zkNode, ex); + continue; + } + } + } + } + + /** + * Initialize Curator framework. + * + * @param zkIpAddr Zookeeper server IP address. + * @param zookeeperPort Zookeeper server port number. + * + * @return CuratorFramework object + */ + private static CuratorFramework initializeCurator(String zkIpAddr, String zookeeperPort) { + + try { + if (!isZooKeeperAccessible(zkIpAddr, zookeeperPort)) { + System.out.println("Unable to connect to Zookeeper"); + LOGGER.log(Level.SEVERE, "Unable to connect to Zookeeper"); + return null; + } + } catch (InterruptedException | IOException ex) { + System.out.println("Unable to connect to Zookeeper"); + LOGGER.log(Level.SEVERE, "Unable to connect to Zookeeper", ex); + return null; + } + + /* + * Connect to ZooKeeper via Curator. + */ + RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3); + String connectString = zkIpAddr + ":" + zookeeperPort; + CuratorFramework curator = CuratorFrameworkFactory.newClient(connectString, SESSION_TIMEOUT_MILLISECONDS, CONNECTION_TIMEOUT_MILLISECONDS, retryPolicy); + curator.start(); + return curator; + } + + /* + * Creates Autopsy coordination service root ZK nodes. + */ + private static Map populateCategoryNodes(CuratorFramework curator) throws KeeperException, CoordinationServiceException { + /* + * Create the top-level root and category nodes. + */ + String rootNodeName = DEFAULT_NAMESPACE_ROOT; + String rootNode = rootNodeName; + + if (!rootNode.startsWith("/")) { + rootNode = "/" + rootNode; + } + Map categoryPaths = new ConcurrentHashMap<>(); + for (CategoryNode node : CategoryNode.values()) { + String nodePath = rootNode + "/" + node.getDisplayName(); + try { + curator.create().creatingParentsIfNeeded().withMode(CreateMode.PERSISTENT).withACL(ZooDefs.Ids.OPEN_ACL_UNSAFE).forPath(nodePath); + } catch (KeeperException ex) { + if (ex.code() != KeeperException.Code.NODEEXISTS) { + throw ex; + } + } catch (Exception ex) { + throw new CoordinationServiceException("Curator experienced an error", ex); + } + categoryPaths.put(node.getDisplayName(), nodePath); + } + return categoryPaths; + } + + /** + * Determines if ZooKeeper is accessible with the current settings. Closes + * the connection prior to returning. + * + * @return true if a connection was achieved, false otherwise + * + * @throws InterruptedException + * @throws IOException + */ + private static boolean isZooKeeperAccessible(String solrIpAddr, String zookeeperPort) throws InterruptedException, IOException { + boolean result = false; + Object workerThreadWaitNotifyLock = new Object(); + String connectString = solrIpAddr + ":" + zookeeperPort; + ZooKeeper zooKeeper = new ZooKeeper(connectString, ZOOKEEPER_SESSION_TIMEOUT_MILLIS, + (WatchedEvent event) -> { + synchronized (workerThreadWaitNotifyLock) { + workerThreadWaitNotifyLock.notifyAll(); + } + }); + synchronized (workerThreadWaitNotifyLock) { + workerThreadWaitNotifyLock.wait(ZOOKEEPER_CONNECTION_TIMEOUT_MILLIS); + } + ZooKeeper.States state = zooKeeper.getState(); + if (state == ZooKeeper.States.CONNECTED || state == ZooKeeper.States.CONNECTEDREADONLY) { + result = true; + } + zooKeeper.close(); + return result; + } + + /** + * Tries to get an exclusive lock on a node path appended to a category path + * in the namespace managed by this coordination service. Returns + * immediately if the lock can not be acquired. + * + * IMPORTANT: The lock needs to be released in the same thread in which it + * is acquired. + * + * @param category The desired category in the namespace. + * @param nodePath The node path to use as the basis for the lock. + * + * @return The lock, or null if the lock could not be obtained. + * + * @throws CoordinationServiceException If there is an error during lock + * acquisition. + */ + private static Lock tryGetExclusiveLock(CuratorFramework curator, CategoryNode category, String nodePath) throws CoordinationServiceException { + String fullNodePath = getFullyQualifiedNodePath(category, nodePath); + try { + InterProcessReadWriteLock lock = new InterProcessReadWriteLock(curator, fullNodePath); + if (!lock.writeLock().acquire(0, TimeUnit.SECONDS)) { + return null; + } + return new Lock(nodePath, lock.writeLock()); + } catch (Exception ex) { + throw new CoordinationServiceException(String.format("Failed to get exclusive lock for %s", fullNodePath), ex); + } + } + + /** + * Retrieve the data associated with the specified node. + * + * @param category The desired category in the namespace. + * @param nodePath The node to retrieve the data for. + * + * @return The data associated with the node, if any, or null if the node + * has not been created yet. + * + * @throws CoordinationServiceException If there is an error setting the + * node data. + * @throws InterruptedException If interrupted while blocked during + * setting of node data. + */ + private static byte[] getNodeData(CategoryNode category, String nodePath) throws CoordinationServiceException, InterruptedException { + String fullNodePath = getFullyQualifiedNodePath(category, nodePath); + try { + return inputCurator.getData().forPath(fullNodePath); + } catch (NoNodeException ex) { + return null; + } catch (Exception ex) { + if (ex instanceof InterruptedException) { + throw (InterruptedException) ex; + } else { + throw new CoordinationServiceException(String.format("Failed to get data for %s", fullNodePath), ex); + } + } + } + + /** + * Store the given data with the specified node. + * + * @param category The desired category in the namespace. + * @param nodePath The node to associate the data with. + * @param data The data to store with the node. + * + * @throws CoordinationServiceException If there is an error setting the + * node data. + * @throws InterruptedException If interrupted while blocked during + * setting of node data. + */ + private static void setNodeData(CuratorFramework curator, CategoryNode category, String nodePath, byte[] data) throws CoordinationServiceException, InterruptedException { + String fullNodePath = getFullyQualifiedNodePath(category, nodePath); + try { + curator.setData().forPath(fullNodePath, data); + } catch (Exception ex) { + if (ex instanceof InterruptedException) { + throw (InterruptedException) ex; + } else { + throw new CoordinationServiceException(String.format("Failed to set data for %s", fullNodePath), ex); + } + } + } + + /** + * Delete the specified node. + * + * @param category The desired category in the namespace. + * @param nodePath The node to delete. + * + * @throws CoordinationServiceException If there is an error setting the + * node data. + * @throws InterruptedException If interrupted while blocked during + * setting of node data. + */ + private static void deleteNode(CategoryNode category, String nodePath) throws CoordinationServiceException, InterruptedException { + String fullNodePath = getFullyQualifiedNodePath(category, nodePath); + try { + inputCurator.delete().forPath(fullNodePath); + } catch (Exception ex) { + if (ex instanceof InterruptedException) { + throw (InterruptedException) ex; + } else { + throw new CoordinationServiceException(String.format("Failed to set data for %s", fullNodePath), ex); + } + } + } + + /** + * Gets a list of the child nodes of a category in the namespace. + * + * @param category The desired category in the namespace. + * + * @return A list of child node names. + * + * @throws CoordinationServiceException If there is an error getting the + * node list. + */ + private static List getNodeList(CategoryNode category) throws CoordinationServiceException { + try { + List list = inputCurator.getChildren().forPath(categoryNodeToPath.get(category.getDisplayName())); + return list; + } catch (Exception ex) { + throw new CoordinationServiceException(String.format("Failed to get node list for %s", category.getDisplayName()), ex); + } + } + + /** + * Creates a node path within a given category. + * + * @param category A category node. + * @param nodePath A node path relative to a category node path. + * + * @return + */ + private static String getFullyQualifiedNodePath(CategoryNode category, String nodePath) { + // nodePath on Unix systems starts with a "/" and ZooKeeper doesn't like two slashes in a row + if (nodePath.startsWith("/")) { + return categoryNodeToPath.get(category.getDisplayName()) + nodePath.toUpperCase(); + } else { + return categoryNodeToPath.get(category.getDisplayName()) + "/" + nodePath.toUpperCase(); + } + } + + /** + * Exception type thrown by the coordination service. + */ + private final static class CoordinationServiceException extends Exception { + + private static final long serialVersionUID = 1L; + + private CoordinationServiceException(String message) { + super(message); + } + + private CoordinationServiceException(String message, Throwable cause) { + super(message, cause); + } + } + + /** + * An opaque encapsulation of a lock for use in distributed synchronization. + * Instances are obtained by calling a get lock method and must be passed to + * a release lock method. + */ + private static class Lock implements AutoCloseable { + + /** + * This implementation uses the Curator read/write lock. see + * http://curator.apache.org/curator-recipes/shared-reentrant-read-write-lock.html + */ + private final InterProcessMutex interProcessLock; + private final String nodePath; + + private Lock(String nodePath, InterProcessMutex lock) { + this.nodePath = nodePath; + this.interProcessLock = lock; + } + + public String getNodePath() { + return nodePath; + } + + public void release() throws CoordinationServiceException { + try { + this.interProcessLock.release(); + } catch (Exception ex) { + throw new CoordinationServiceException(String.format("Failed to release the lock on %s", nodePath), ex); + } + } + + @Override + public void close() throws CoordinationServiceException { + release(); + } + } + + /** + * Category nodes are the immediate children of the root node of a shared + * hierarchical namespace managed by a coordination service. + */ + public enum CategoryNode { + + CASES("cases"), + MANIFESTS("manifests"), + CONFIG("config"), + CENTRAL_REPO("centralRepository"), + HEALTH_MONITOR("healthMonitor"); + + private final String displayName; + + private CategoryNode(String displayName) { + this.displayName = displayName; + } + + public String getDisplayName() { + return displayName; + } + } +} diff --git a/build.xml b/build.xml index a7c64ef4ba..7377d59c04 100644 --- a/build.xml +++ b/build.xml @@ -106,8 +106,7 @@ - - + @@ -115,6 +114,14 @@ + + + + + + + + diff --git a/docs/doxygen-user/images/mt_config.png b/docs/doxygen-user/images/mt_config.png index e3022aec0d..b793191ae8 100644 Binary files a/docs/doxygen-user/images/mt_config.png and b/docs/doxygen-user/images/mt_config.png differ diff --git a/docs/doxygen-user/images/mt_ocr_image.png b/docs/doxygen-user/images/mt_ocr_image.png new file mode 100644 index 0000000000..b4e16bfa09 Binary files /dev/null and b/docs/doxygen-user/images/mt_ocr_image.png differ diff --git a/docs/doxygen-user/images/mt_ocr_result.png b/docs/doxygen-user/images/mt_ocr_result.png new file mode 100644 index 0000000000..27d9af77f0 Binary files /dev/null and b/docs/doxygen-user/images/mt_ocr_result.png differ diff --git a/docs/doxygen-user/keyword_search.dox b/docs/doxygen-user/keyword_search.dox index 271e660cac..fd207a6de2 100644 --- a/docs/doxygen-user/keyword_search.dox +++ b/docs/doxygen-user/keyword_search.dox @@ -57,6 +57,7 @@ The "Indexed Text" tab shows the results when running the keyword search module \image html keyword-search-ocr-indexed-text.png +\anchor keyword_search_ocr_config By default, OCR is only configured for English text. Its configuration depends on the presence of language files (called "traineddata" files) that exist in a location that Autopsy can understand. To add support for more languages, you will need to download additional "traineddata" and move them to the right location. The following steps breakdown this process for you: diff --git a/docs/doxygen-user/machine_translation.dox b/docs/doxygen-user/machine_translation.dox index cec795d932..4609c81257 100644 --- a/docs/doxygen-user/machine_translation.dox +++ b/docs/doxygen-user/machine_translation.dox @@ -17,6 +17,8 @@ To set up a machine translation service, go to Options->Tools and then select th Each service will require slightly different configuration steps. After setting everything up, you can run a quick check that the service is set up correctly using the "Test" button. +The checkbox at the bottom allows you to enable or disable optical character recognition (OCR). When enabled, if you select an image in the \ref mt_content_viewer "content viewer" Autopsy will use OCR to attempt to extract text to be translated. Instructions for installing OCR packages for different languages can be found on the \ref keyword_search_ocr_config "Keyword Search page". + \section mt_file_names Translating File Names You can use machine translation to automatically translate file and folder names, such as the ones seen below: @@ -49,4 +51,12 @@ Then use the drop-down menu on the right to change from "Original Text" to "Tran \image html mt_message_translated.png +If you've enabled OCR as described in the \ref mt_config section above, you can extract and translate text from images. Here is an image containing the beginning of a French poem: + +\image html mt_ocr_image.png + +If you go to the Text tab and then the Translation viewer it will use OCR to read text from the image and then display the translation. + +\image html mt_ocr_result.png + */ \ No newline at end of file diff --git a/thunderbirdparser/nbproject/project.xml b/thunderbirdparser/nbproject/project.xml index 4c32aa3861..31276d7838 100644 --- a/thunderbirdparser/nbproject/project.xml +++ b/thunderbirdparser/nbproject/project.xml @@ -44,7 +44,8 @@ - 1.0 + 1 + 23 diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java index 0f9759819f..9957c52454 100644 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java @@ -79,7 +79,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { private static final int MBOX_SIZE_TO_SPLIT = 1048576000; private Case currentCase; - + /** * Empty constructor. */ @@ -139,6 +139,10 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { boolean isPstFile = PstParser.isPstFile(abstractFile); boolean isVcardFile = VcardParser.isVcardFile(abstractFile); + if (context.fileIngestIsCancelled()) { + return ProcessResult.OK; + } + if (isMbox || isEMLFile || isPstFile || isVcardFile ) { try { communicationArtifactsHelper = new CommunicationArtifactsHelper(currentCase.getSleuthkitCase(), @@ -213,6 +217,9 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { Iterator pstMsgIterator = parser.getEmailMessageIterator(); if (pstMsgIterator != null) { processEmails(parser.getPartialEmailMessages(), pstMsgIterator , abstractFile); + if (context.fileIngestIsCancelled()) { + return ProcessResult.OK; + } } else { // sometimes parser returns ParseResult=OK but there are no messages postErrorMessage( @@ -321,6 +328,9 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { } processMboxFile(file, abstractFile, emailFolder); + if (context.fileIngestIsCancelled()) { + return ProcessResult.OK; + } if (file.delete() == false) { logger.log(Level.INFO, "Failed to delete temp file: {0}", file.getName()); //NON-NLS @@ -349,7 +359,9 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { if (splitFile.delete() == false) { logger.log(Level.INFO, "Failed to delete temp file: {0}", splitFile); //NON-NLS } - + if (context.fileIngestIsCancelled()) { + return ProcessResult.OK; + } } } @@ -385,6 +397,9 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { List emails = new ArrayList<>(); if(emailIterator != null) { while(emailIterator.hasNext()) { + if (context.fileIngestIsCancelled()) { + return; + } EmailMessage emailMessage = emailIterator.next(); if(emailMessage != null) { emails.add(emailMessage); @@ -526,6 +541,10 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { int msgCnt = 0; while(fullMessageIterator.hasNext()) { + if (context.fileIngestIsCancelled()) { + return; + } + EmailMessage current = fullMessageIterator.next(); if(current == null) { @@ -550,6 +569,9 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { if (derivedFiles.isEmpty() == false) { for (AbstractFile derived : derivedFiles) { + if (context.fileIngestIsCancelled()) { + return; + } services.fireModuleContentEvent(new ModuleContentEvent(derived)); } } @@ -675,6 +697,10 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { String senderAddress; senderAddressList.addAll(findEmailAddresess(from)); + if (context.fileIngestIsCancelled()) { + return null; + } + AccountFileInstance senderAccountInstance = null; if (senderAddressList.size() == 1) { @@ -690,13 +716,20 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { logger.log(Level.WARNING, "Failed to find sender address, from = {0}", from); //NON-NLS } + if (context.fileIngestIsCancelled()) { + return null; + } + List recipientAddresses = new ArrayList<>(); recipientAddresses.addAll(findEmailAddresess(to)); recipientAddresses.addAll(findEmailAddresess(cc)); recipientAddresses.addAll(findEmailAddresess(bcc)); List recipientAccountInstances = new ArrayList<>(); - recipientAddresses.forEach((addr) -> { + for (String addr : recipientAddresses) { + if (context.fileIngestIsCancelled()) { + return null; + } try { AccountFileInstance recipientAccountInstance = currentCase.getSleuthkitCase().getCommunicationsManager().createAccountFileInstance(Account.Type.EMAIL, addr, @@ -706,7 +739,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { catch(TskCoreException ex) { logger.log(Level.WARNING, "Failed to create account for email address " + addr, ex); //NON-NLS } - }); + } addArtifactAttribute(headers, ATTRIBUTE_TYPE.TSK_HEADERS, bbattributes); addArtifactAttribute(from, ATTRIBUTE_TYPE.TSK_EMAIL_FROM, bbattributes); @@ -731,12 +764,23 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { try { + if (context.fileIngestIsCancelled()) { + return null; + } bbart = abstractFile.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_EMAIL_MSG); bbart.addAttributes(bbattributes); + if (context.fileIngestIsCancelled()) { + return null; + } + // Add account relationships currentCase.getSleuthkitCase().getCommunicationsManager().addRelationships(senderAccountInstance, recipientAccountInstances, bbart,Relationship.Type.MESSAGE, dateL); + + if (context.fileIngestIsCancelled()) { + return null; + } try { // index the artifact for keyword search