diff --git a/Core/nbproject/project.properties b/Core/nbproject/project.properties index 69d3afa92d..984792fc91 100644 --- a/Core/nbproject/project.properties +++ b/Core/nbproject/project.properties @@ -1,5 +1,10 @@ +file.reference.jdom-2.0.5-contrib.jar=release/modules/ext/jdom-2.0.5-contrib.jar +file.reference.jdom-2.0.5.jar=release/modules/ext/jdom-2.0.5.jar +file.reference.metadata-extractor-2.6.2.jar=release/modules/ext/metadata-extractor-2.6.2.jar file.reference.sqlite-jdbc-3.8.0-SNAPSHOT.jar=release/modules/ext/sqlite-jdbc-3.8.0-SNAPSHOT.jar +file.reference.tika-core-1.2.jar=release/modules/ext/tika-core-1.2.jar file.reference.Tsk_DataModel.jar=release/modules/ext/Tsk_DataModel.jar +file.reference.xmpcore.jar=release/modules/ext/xmpcore.jar javac.source=1.7 javac.compilerargs=-Xlint -Xlint:-serial license.file=../LICENSE-2.0.txt diff --git a/Core/nbproject/project.xml b/Core/nbproject/project.xml index 7dd5bfd278..a70456a44a 100644 --- a/Core/nbproject/project.xml +++ b/Core/nbproject/project.xml @@ -207,6 +207,26 @@ org.sleuthkit.autopsy.report org.sleuthkit.datamodel + + ext/jdom-2.0.5.jar + release/modules/ext/jdom-2.0.5.jar + + + ext/sevenzipjbinding.jar + release/modules/ext/sevenzipjbinding.jar + + + ext/xmpcore.jar + release/modules/ext/xmpcore.jar + + + ext/jdom-2.0.5-contrib.jar + release/modules/ext/jdom-2.0.5-contrib.jar + + + ext/sevenzipjbinding-AllPlatforms.jar + release/modules/ext/sevenzipjbinding-AllPlatforms.jar + ext/Tsk_DataModel.jar release/modules/ext/Tsk_DataModel.jar @@ -215,6 +235,14 @@ ext/sqlite-jdbc-3.8.0-SNAPSHOT.jar release/modules/ext/sqlite-jdbc-3.8.0-SNAPSHOT.jar + + ext/metadata-extractor-2.6.2.jar + release/modules/ext/metadata-extractor-2.6.2.jar + + + ext/tika-core-1.2.jar + release/modules/ext/tika-core-1.2.jar + diff --git a/Timeline/release/mactime/mactime.exe b/Core/release/mactime/mactime.exe similarity index 100% rename from Timeline/release/mactime/mactime.exe rename to Core/release/mactime/mactime.exe diff --git a/ExifParser/release/modules/ext/LICENSE-2.0.txt b/Core/release/modules/ext/LICENSE-2.0.txt similarity index 100% rename from ExifParser/release/modules/ext/LICENSE-2.0.txt rename to Core/release/modules/ext/LICENSE-2.0.txt diff --git a/ExifParser/release/modules/ext/README.txt b/Core/release/modules/ext/README.txt similarity index 100% rename from ExifParser/release/modules/ext/README.txt rename to Core/release/modules/ext/README.txt diff --git a/Core/release/modules/ext/jdom-2.0.5-contrib.jar b/Core/release/modules/ext/jdom-2.0.5-contrib.jar new file mode 100644 index 0000000000..7432b751a3 Binary files /dev/null and b/Core/release/modules/ext/jdom-2.0.5-contrib.jar differ diff --git a/Core/release/modules/ext/jdom-2.0.5.jar b/Core/release/modules/ext/jdom-2.0.5.jar new file mode 100644 index 0000000000..b6996c725a Binary files /dev/null and b/Core/release/modules/ext/jdom-2.0.5.jar differ diff --git a/ExifParser/release/modules/ext/metadata-extractor-2.6.2.jar b/Core/release/modules/ext/metadata-extractor-2.6.2.jar old mode 100644 new mode 100755 similarity index 100% rename from ExifParser/release/modules/ext/metadata-extractor-2.6.2.jar rename to Core/release/modules/ext/metadata-extractor-2.6.2.jar diff --git a/SevenZip/release/modules/ext/sevenzipjbinding-AllPlatforms.jar b/Core/release/modules/ext/sevenzipjbinding-AllPlatforms.jar old mode 100644 new mode 100755 similarity index 100% rename from SevenZip/release/modules/ext/sevenzipjbinding-AllPlatforms.jar rename to Core/release/modules/ext/sevenzipjbinding-AllPlatforms.jar diff --git a/SevenZip/release/modules/ext/sevenzipjbinding.jar b/Core/release/modules/ext/sevenzipjbinding.jar old mode 100644 new mode 100755 similarity index 100% rename from SevenZip/release/modules/ext/sevenzipjbinding.jar rename to Core/release/modules/ext/sevenzipjbinding.jar diff --git a/FileTypeId/release/modules/ext/tika-core-1.2.jar b/Core/release/modules/ext/tika-core-1.2.jar old mode 100644 new mode 100755 similarity index 100% rename from FileTypeId/release/modules/ext/tika-core-1.2.jar rename to Core/release/modules/ext/tika-core-1.2.jar diff --git a/ExifParser/release/modules/ext/xmpcore.jar b/Core/release/modules/ext/xmpcore.jar old mode 100644 new mode 100755 similarity index 100% rename from ExifParser/release/modules/ext/xmpcore.jar rename to Core/release/modules/ext/xmpcore.jar diff --git a/Core/src/org/sleuthkit/autopsy/actions/Bundle.properties b/Core/src/org/sleuthkit/autopsy/actions/Bundle.properties index 2151fafa49..cf80037c2c 100755 --- a/Core/src/org/sleuthkit/autopsy/actions/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/actions/Bundle.properties @@ -48,3 +48,4 @@ GetTagNameDialog.unableToAddTagNameToCase.msg=Unable to add the {0} tag name to GetTagNameDialog.taggingErr=Tagging Error GetTagNameDialog.tagNameAlreadyDef.msg=A {0} tag name has already been defined. GetTagNameDialog.dupTagErr=Duplicate Tag Error +OpenLogFolder.error1=Log File Not Found: {0} diff --git a/Core/src/org/sleuthkit/autopsy/actions/OpenLogFolderAction.java b/Core/src/org/sleuthkit/autopsy/actions/OpenLogFolderAction.java new file mode 100755 index 0000000000..9276a747f9 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/actions/OpenLogFolderAction.java @@ -0,0 +1,64 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2014 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.actions; + +import java.awt.event.ActionEvent; +import java.awt.event.ActionListener; +import java.awt.Desktop; +import java.io.File; +import java.io.IOException; +import org.openide.DialogDisplayer; +import org.openide.NotifyDescriptor; +import org.openide.awt.ActionID; +import org.openide.awt.ActionReference; +import org.openide.awt.ActionRegistration; +import org.openide.modules.Places; +import org.openide.util.Exceptions; +import org.openide.util.NbBundle.Messages; + +@ActionID( + category = "Help", + id = "org.sleuthkit.autopsy.actions.OpenLogFolder") +@ActionRegistration( + displayName = "#CTL_OpenLogFolder") +@ActionReference(path = "Menu/Help", position = 1750) +@Messages("CTL_OpenLogFolder=Open Log Folder") +/** + * Action in menu to open the folder containing the log files + */ +public final class OpenLogFolderAction implements ActionListener { + + @Override + public void actionPerformed(ActionEvent e) { + try { + File logDir = new File(Places.getUserDirectory().getAbsolutePath() + File.separator + "var" + File.separator + "log"); + if (logDir.exists() == false) { + NotifyDescriptor d = + new NotifyDescriptor.Message( + java.text.MessageFormat.format(java.util.ResourceBundle.getBundle("org/sleuthkit/autopsy/actions/Bundle").getString("OpenLogFolder.error1"), new Object[]{logDir.getAbsolutePath()}), + NotifyDescriptor.ERROR_MESSAGE); + DialogDisplayer.getDefault().notify(d); + } else { + Desktop.getDesktop().open(logDir); + } + } catch (IOException ex) { + Exceptions.printStackTrace(ex); + } + } +} diff --git a/Core/src/org/sleuthkit/autopsy/core/layer.xml b/Core/src/org/sleuthkit/autopsy/core/layer.xml index 6e8d10bac8..0a0d481991 100644 --- a/Core/src/org/sleuthkit/autopsy/core/layer.xml +++ b/Core/src/org/sleuthkit/autopsy/core/layer.xml @@ -122,8 +122,13 @@ - - + + + + + + + @@ -260,7 +265,7 @@ - + @@ -317,6 +322,12 @@ + + + + + + - - - + + + - + diff --git a/ExifParser/src/org/sleuthkit/autopsy/exifparser/Bundle.properties b/Core/src/org/sleuthkit/autopsy/modules/exif/Bundle.properties similarity index 100% rename from ExifParser/src/org/sleuthkit/autopsy/exifparser/Bundle.properties rename to Core/src/org/sleuthkit/autopsy/modules/exif/Bundle.properties diff --git a/ExifParser/src/org/sleuthkit/autopsy/exifparser/Bundle_ja.properties b/Core/src/org/sleuthkit/autopsy/modules/exif/Bundle_ja.properties similarity index 100% rename from ExifParser/src/org/sleuthkit/autopsy/exifparser/Bundle_ja.properties rename to Core/src/org/sleuthkit/autopsy/modules/exif/Bundle_ja.properties diff --git a/ExifParser/src/org/sleuthkit/autopsy/exifparser/ExifParserFileIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/exif/ExifParserFileIngestModule.java similarity index 86% rename from ExifParser/src/org/sleuthkit/autopsy/exifparser/ExifParserFileIngestModule.java rename to Core/src/org/sleuthkit/autopsy/modules/exif/ExifParserFileIngestModule.java index 58656cdc80..6e9f8c4ecf 100644 --- a/ExifParser/src/org/sleuthkit/autopsy/exifparser/ExifParserFileIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/exif/ExifParserFileIngestModule.java @@ -16,7 +16,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.sleuthkit.autopsy.exifparser; +package org.sleuthkit.autopsy.modules.exif; import com.drew.imaging.ImageMetadataReader; import com.drew.imaging.ImageProcessingException; @@ -32,13 +32,16 @@ import java.io.InputStream; import java.util.ArrayList; import java.util.Collection; import java.util.Date; +import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Level; import org.sleuthkit.autopsy.coreutils.ImageUtils; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.ingest.IngestModuleAdapter; import org.sleuthkit.autopsy.ingest.FileIngestModule; +import org.sleuthkit.autopsy.ingest.IngestJobContext; import org.sleuthkit.autopsy.ingest.IngestServices; import org.sleuthkit.autopsy.ingest.ModuleDataEvent; +import org.sleuthkit.autopsy.ingest.IngestModuleReferenceCounter; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardAttribute; @@ -57,12 +60,21 @@ public final class ExifParserFileIngestModule extends IngestModuleAdapter implem private static final Logger logger = Logger.getLogger(ExifParserFileIngestModule.class.getName()); private final IngestServices services = IngestServices.getInstance(); - private int filesProcessed = 0; - private boolean filesToFire = false; - + private AtomicInteger filesProcessed = new AtomicInteger(0); + private volatile boolean filesToFire = false; + private long jobId; + private static final IngestModuleReferenceCounter refCounter = new IngestModuleReferenceCounter(); + ExifParserFileIngestModule() { } + @Override + public void startUp(IngestJobContext context) throws IngestModuleException { + jobId = context.getJobId(); + refCounter.incrementAndGet(jobId); + } + + @Override public ProcessResult process(AbstractFile content) { //skip unalloc @@ -76,8 +88,8 @@ public final class ExifParserFileIngestModule extends IngestModuleAdapter implem } // update the tree every 1000 files if we have EXIF data that is not being being displayed - filesProcessed++; - if ((filesToFire) && (filesProcessed % 1000 == 0)) { + final int filesProcessedValue = filesProcessed.incrementAndGet(); + if ((filesToFire) && (filesProcessedValue % 1000 == 0)) { services.fireModuleDataEvent(new ModuleDataEvent(ExifParserModuleFactory.getModuleName(), BlackboardArtifact.ARTIFACT_TYPE.TSK_METADATA_EXIF)); filesToFire = false; } @@ -187,9 +199,12 @@ public final class ExifParserFileIngestModule extends IngestModuleAdapter implem @Override public void shutDown(boolean ingestJobCancelled) { - if (filesToFire) { - //send the final new data event - services.fireModuleDataEvent(new ModuleDataEvent(ExifParserModuleFactory.getModuleName(), BlackboardArtifact.ARTIFACT_TYPE.TSK_METADATA_EXIF)); + // We only need to check for this final event on the last module per job + if (refCounter.decrementAndGet(jobId) == 0) { + if (filesToFire) { + //send the final new data event + services.fireModuleDataEvent(new ModuleDataEvent(ExifParserModuleFactory.getModuleName(), BlackboardArtifact.ARTIFACT_TYPE.TSK_METADATA_EXIF)); + } } } } \ No newline at end of file diff --git a/ExifParser/src/org/sleuthkit/autopsy/exifparser/ExifParserModuleFactory.java b/Core/src/org/sleuthkit/autopsy/modules/exif/ExifParserModuleFactory.java similarity index 97% rename from ExifParser/src/org/sleuthkit/autopsy/exifparser/ExifParserModuleFactory.java rename to Core/src/org/sleuthkit/autopsy/modules/exif/ExifParserModuleFactory.java index 73078e67cb..8f1b1a4efe 100755 --- a/ExifParser/src/org/sleuthkit/autopsy/exifparser/ExifParserModuleFactory.java +++ b/Core/src/org/sleuthkit/autopsy/modules/exif/ExifParserModuleFactory.java @@ -16,7 +16,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.sleuthkit.autopsy.exifparser; +package org.sleuthkit.autopsy.modules.exif; import org.openide.util.lookup.ServiceProvider; import org.sleuthkit.autopsy.coreutils.Version; diff --git a/ExifParser/src/org/sleuthkit/autopsy/exifparser/layer.xml b/Core/src/org/sleuthkit/autopsy/modules/exif/layer.xml similarity index 100% rename from ExifParser/src/org/sleuthkit/autopsy/exifparser/layer.xml rename to Core/src/org/sleuthkit/autopsy/modules/exif/layer.xml diff --git a/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/AddFileExtensionAction.java b/Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/AddFileExtensionAction.java similarity index 97% rename from FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/AddFileExtensionAction.java rename to Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/AddFileExtensionAction.java index 463178e25e..7be2e30983 100644 --- a/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/AddFileExtensionAction.java +++ b/Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/AddFileExtensionAction.java @@ -16,7 +16,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.sleuthkit.autopsy.fileextmismatch; +package org.sleuthkit.autopsy.modules.fileextmismatch; import org.openide.util.NbBundle; import java.awt.event.ActionEvent; diff --git a/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/Bundle.properties b/Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/Bundle.properties similarity index 97% rename from FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/Bundle.properties rename to Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/Bundle.properties index be85efacb9..c5610f46ad 100644 --- a/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/Bundle.properties @@ -34,7 +34,6 @@ FileExtMismatchIngestModule.complete.totalFiles=Total Files Processed FileExtMismatchIngestModule.complete.svcMsg.text=File Extension Mismatch Results FileExtMismatchOptionsPanelController.moduleErr=Module Error FileExtMismatchOptionsPanelController.moduleErr.msg=A module caused an error listening to FileExtMismatchOptionsPanelController updates. See log to determine which module. Some data could be incomplete. -FileExtMismatchModuleSettingsPanel.skipKnownFilesCheckbox.text=Skip known files (NSRL) FileExtMismatchModuleSettingsPanel.skipTextPlain.text=Skip text files FileExtMismatchModuleSettingsPanel.skipNoExtCheckBox.text=Skip files without extensions FileExtMismatchSettingsPanel.addTypeButton.text=Add Type diff --git a/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/Bundle_ja.properties b/Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/Bundle_ja.properties similarity index 100% rename from FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/Bundle_ja.properties rename to Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/Bundle_ja.properties diff --git a/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchContextMenuActionsProvider.java b/Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/FileExtMismatchContextMenuActionsProvider.java similarity index 99% rename from FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchContextMenuActionsProvider.java rename to Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/FileExtMismatchContextMenuActionsProvider.java index 73907a04ac..b52140876f 100644 --- a/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchContextMenuActionsProvider.java +++ b/Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/FileExtMismatchContextMenuActionsProvider.java @@ -18,7 +18,7 @@ */ -package org.sleuthkit.autopsy.fileextmismatch; +package org.sleuthkit.autopsy.modules.fileextmismatch; import java.util.ArrayList; import java.util.Arrays; diff --git a/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchDetectorModuleFactory.java b/Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/FileExtMismatchDetectorModuleFactory.java similarity index 98% rename from FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchDetectorModuleFactory.java rename to Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/FileExtMismatchDetectorModuleFactory.java index 855c586870..645de2ba48 100755 --- a/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchDetectorModuleFactory.java +++ b/Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/FileExtMismatchDetectorModuleFactory.java @@ -16,7 +16,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.sleuthkit.autopsy.fileextmismatch; +package org.sleuthkit.autopsy.modules.fileextmismatch; import org.openide.util.NbBundle; import org.openide.util.lookup.ServiceProvider; diff --git a/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchDetectorModuleSettings.java b/Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/FileExtMismatchDetectorModuleSettings.java similarity index 86% rename from FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchDetectorModuleSettings.java rename to Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/FileExtMismatchDetectorModuleSettings.java index 66d0ba9826..e01d1856f0 100755 --- a/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchDetectorModuleSettings.java +++ b/Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/FileExtMismatchDetectorModuleSettings.java @@ -16,7 +16,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.sleuthkit.autopsy.fileextmismatch; +package org.sleuthkit.autopsy.modules.fileextmismatch; import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettings; @@ -25,7 +25,6 @@ import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettings; */ final class FileExtMismatchDetectorModuleSettings implements IngestModuleIngestJobSettings { - private boolean skipKnownFiles = false; private boolean skipFilesWithNoExtension = true; private boolean skipFilesWithTextPlainMimeType = false; @@ -33,19 +32,10 @@ final class FileExtMismatchDetectorModuleSettings implements IngestModuleIngestJ } FileExtMismatchDetectorModuleSettings(boolean skipKnownFiles, boolean skipFilesWithNoExtension, boolean skipFilesWithTextPlainMimeType) { - this.skipKnownFiles = skipKnownFiles; this.skipFilesWithNoExtension = skipFilesWithNoExtension; this.skipFilesWithTextPlainMimeType = skipFilesWithTextPlainMimeType; } - void setSkipKnownFiles(boolean enabled) { - skipKnownFiles = enabled; - } - - boolean skipKnownFiles() { - return skipKnownFiles; - } - void setSkipFilesWithNoExtension(boolean enabled) { skipFilesWithNoExtension = enabled; } diff --git a/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/FileExtMismatchIngestModule.java similarity index 76% rename from FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchIngestModule.java rename to Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/FileExtMismatchIngestModule.java index 417a47cb24..5745017a08 100644 --- a/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/FileExtMismatchIngestModule.java @@ -16,13 +16,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.sleuthkit.autopsy.fileextmismatch; +package org.sleuthkit.autopsy.modules.fileextmismatch; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.concurrent.atomic.AtomicLong; import java.util.logging.Level; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.coreutils.Logger; @@ -32,13 +33,13 @@ import org.sleuthkit.autopsy.ingest.IngestJobContext; import org.sleuthkit.autopsy.ingest.IngestMessage; import org.sleuthkit.autopsy.ingest.IngestServices; import org.sleuthkit.autopsy.ingest.ModuleDataEvent; +import org.sleuthkit.autopsy.ingest.IngestModuleReferenceCounter; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE; import org.sleuthkit.datamodel.BlackboardAttribute; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskData; -import org.sleuthkit.datamodel.TskData.FileKnown; import org.sleuthkit.datamodel.TskException; /** @@ -47,12 +48,13 @@ import org.sleuthkit.datamodel.TskException; public class FileExtMismatchIngestModule extends IngestModuleAdapter implements FileIngestModule { private static final Logger logger = Logger.getLogger(FileExtMismatchIngestModule.class.getName()); - private static int messageId = 0; // RJCTODO: This is not thread safe private final IngestServices services = IngestServices.getInstance(); private final FileExtMismatchDetectorModuleSettings settings; private HashMap SigTypeToExtMap = new HashMap<>(); - private long processTime = 0; - private long numFiles = 0; + private long jobId; + private static AtomicLong processTime = new AtomicLong(0); + private static AtomicLong numFiles = new AtomicLong(0); + private static final IngestModuleReferenceCounter refCounter = new IngestModuleReferenceCounter(); FileExtMismatchIngestModule(FileExtMismatchDetectorModuleSettings settings) { this.settings = settings; @@ -60,6 +62,8 @@ public class FileExtMismatchIngestModule extends IngestModuleAdapter implements @Override public void startUp(IngestJobContext context) throws IngestModuleException { + jobId = context.getJobId(); + refCounter.incrementAndGet(jobId); FileExtMismatchXML xmlLoader = FileExtMismatchXML.getDefault(); SigTypeToExtMap = xmlLoader.load(); } @@ -78,17 +82,13 @@ public class FileExtMismatchIngestModule extends IngestModuleAdapter implements return ProcessResult.OK; } - if (settings.skipKnownFiles() && (abstractFile.getKnown() == FileKnown.KNOWN)) { - return ProcessResult.OK; - } - try { long startTime = System.currentTimeMillis(); boolean mismatchDetected = compareSigTypeToExt(abstractFile); - processTime += (System.currentTimeMillis() - startTime); - numFiles++; + processTime.getAndAdd(System.currentTimeMillis() - startTime); + numFiles.getAndIncrement(); if (mismatchDetected) { // add artifact @@ -154,19 +154,22 @@ public class FileExtMismatchIngestModule extends IngestModuleAdapter implements @Override public void shutDown(boolean ingestJobCancelled) { - StringBuilder detailsSb = new StringBuilder(); - detailsSb.append(""); - detailsSb.append(""); - detailsSb.append("\n"); - detailsSb.append("\n"); - detailsSb.append("
").append(FileExtMismatchDetectorModuleFactory.getModuleName()).append("
").append( - NbBundle.getMessage(this.getClass(), "FileExtMismatchIngestModule.complete.totalProcTime")) - .append("").append(processTime).append("
").append( - NbBundle.getMessage(this.getClass(), "FileExtMismatchIngestModule.complete.totalFiles")) - .append("").append(numFiles).append("
"); - services.postMessage(IngestMessage.createMessage(++messageId, IngestMessage.MessageType.INFO, FileExtMismatchDetectorModuleFactory.getModuleName(), - NbBundle.getMessage(this.getClass(), - "FileExtMismatchIngestModule.complete.svcMsg.text"), - detailsSb.toString())); + // We only need to post the summary msg from the last module per job + if (refCounter.decrementAndGet(jobId) == 0) { + StringBuilder detailsSb = new StringBuilder(); + detailsSb.append(""); + detailsSb.append(""); + detailsSb.append("\n"); + detailsSb.append("\n"); + detailsSb.append("
").append(FileExtMismatchDetectorModuleFactory.getModuleName()).append("
").append( + NbBundle.getMessage(this.getClass(), "FileExtMismatchIngestModule.complete.totalProcTime")) + .append("").append(processTime.get()).append("
").append( + NbBundle.getMessage(this.getClass(), "FileExtMismatchIngestModule.complete.totalFiles")) + .append("").append(numFiles.get()).append("
"); + services.postMessage(IngestMessage.createMessage(IngestMessage.MessageType.INFO, FileExtMismatchDetectorModuleFactory.getModuleName(), + NbBundle.getMessage(this.getClass(), + "FileExtMismatchIngestModule.complete.svcMsg.text"), + detailsSb.toString())); + } } } diff --git a/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchModuleSettingsPanel.form b/Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/FileExtMismatchModuleSettingsPanel.form similarity index 67% rename from FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchModuleSettingsPanel.form rename to Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/FileExtMismatchModuleSettingsPanel.form index a2e7553626..be70dbe504 100644 --- a/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchModuleSettingsPanel.form +++ b/Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/FileExtMismatchModuleSettingsPanel.form @@ -21,7 +21,6 @@ - @@ -33,9 +32,7 @@ - - - + @@ -45,7 +42,7 @@ - + @@ -55,7 +52,7 @@ - + @@ -65,15 +62,5 @@ - - - - - - - - - - diff --git a/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchModuleSettingsPanel.java b/Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/FileExtMismatchModuleSettingsPanel.java similarity index 78% rename from FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchModuleSettingsPanel.java rename to Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/FileExtMismatchModuleSettingsPanel.java index 5dc0c5321d..de619e3510 100644 --- a/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchModuleSettingsPanel.java +++ b/Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/FileExtMismatchModuleSettingsPanel.java @@ -16,7 +16,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.sleuthkit.autopsy.fileextmismatch; +package org.sleuthkit.autopsy.modules.fileextmismatch; import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettings; import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettingsPanel; @@ -38,7 +38,6 @@ final class FileExtMismatchModuleSettingsPanel extends IngestModuleIngestJobSett private void customizeComponents() { skipNoExtCheckBox.setSelected(settings.skipFilesWithNoExtension()); skipTextPlain.setSelected(settings.skipFilesWithTextPlainMimeType()); - skipKnownFilesCheckbox.setSelected(settings.skipKnownFiles()); } @Override @@ -57,7 +56,6 @@ final class FileExtMismatchModuleSettingsPanel extends IngestModuleIngestJobSett skipNoExtCheckBox = new javax.swing.JCheckBox(); skipTextPlain = new javax.swing.JCheckBox(); - skipKnownFilesCheckbox = new javax.swing.JCheckBox(); skipNoExtCheckBox.setSelected(true); skipNoExtCheckBox.setText(org.openide.util.NbBundle.getMessage(FileExtMismatchModuleSettingsPanel.class, "FileExtMismatchModuleSettingsPanel.skipNoExtCheckBox.text")); // NOI18N @@ -75,13 +73,6 @@ final class FileExtMismatchModuleSettingsPanel extends IngestModuleIngestJobSett } }); - skipKnownFilesCheckbox.setText(org.openide.util.NbBundle.getMessage(FileExtMismatchModuleSettingsPanel.class, "FileExtMismatchModuleSettingsPanel.skipKnownFilesCheckbox.text")); // NOI18N - skipKnownFilesCheckbox.addActionListener(new java.awt.event.ActionListener() { - public void actionPerformed(java.awt.event.ActionEvent evt) { - skipKnownFilesCheckboxActionPerformed(evt); - } - }); - javax.swing.GroupLayout layout = new javax.swing.GroupLayout(this); this.setLayout(layout); layout.setHorizontalGroup( @@ -90,8 +81,7 @@ final class FileExtMismatchModuleSettingsPanel extends IngestModuleIngestJobSett .addContainerGap() .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addComponent(skipTextPlain) - .addComponent(skipNoExtCheckBox) - .addComponent(skipKnownFilesCheckbox)) + .addComponent(skipNoExtCheckBox)) .addGap(0, 138, Short.MAX_VALUE)) ); layout.setVerticalGroup( @@ -100,9 +90,7 @@ final class FileExtMismatchModuleSettingsPanel extends IngestModuleIngestJobSett .addComponent(skipNoExtCheckBox) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) .addComponent(skipTextPlain) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) - .addComponent(skipKnownFilesCheckbox) - .addContainerGap(28, Short.MAX_VALUE)) + .addContainerGap(51, Short.MAX_VALUE)) ); }// //GEN-END:initComponents @@ -114,12 +102,7 @@ final class FileExtMismatchModuleSettingsPanel extends IngestModuleIngestJobSett settings.setSkipFilesWithTextPlainMimeType(skipTextPlain.isSelected()); }//GEN-LAST:event_skipTextPlainActionPerformed - private void skipKnownFilesCheckboxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_skipKnownFilesCheckboxActionPerformed - settings.setSkipKnownFiles(skipKnownFilesCheckbox.isSelected()); - }//GEN-LAST:event_skipKnownFilesCheckboxActionPerformed - // Variables declaration - do not modify//GEN-BEGIN:variables - private javax.swing.JCheckBox skipKnownFilesCheckbox; private javax.swing.JCheckBox skipNoExtCheckBox; private javax.swing.JCheckBox skipTextPlain; // End of variables declaration//GEN-END:variables diff --git a/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchOptionsPanelController.java b/Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/FileExtMismatchOptionsPanelController.java similarity index 96% rename from FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchOptionsPanelController.java rename to Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/FileExtMismatchOptionsPanelController.java index ba4f85d0b5..35749a810e 100644 --- a/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchOptionsPanelController.java +++ b/Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/FileExtMismatchOptionsPanelController.java @@ -2,7 +2,7 @@ * To change this template, choose Tools | Templates * and open the template in the editor. */ -package org.sleuthkit.autopsy.fileextmismatch; +package org.sleuthkit.autopsy.modules.fileextmismatch; import java.beans.PropertyChangeListener; import java.beans.PropertyChangeSupport; @@ -17,7 +17,7 @@ import org.sleuthkit.autopsy.coreutils.Logger; @OptionsPanelController.TopLevelRegistration( categoryName = "#OptionsCategory_Name_FileExtMismatchOptions", - iconBase = "org/sleuthkit/autopsy/fileextmismatch/options-icon.png", + iconBase = "org/sleuthkit/autopsy/modules/fileextmismatch/options-icon.png", position = 4, keywords = "#OptionsCategory_FileExtMismatch", keywordsCategory = "KeywordSearchOptions") diff --git a/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchSettingsPanel.form b/Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/FileExtMismatchSettingsPanel.form similarity index 87% rename from FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchSettingsPanel.form rename to Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/FileExtMismatchSettingsPanel.form index 299bacab53..dc2898697c 100644 --- a/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchSettingsPanel.form +++ b/Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/FileExtMismatchSettingsPanel.form @@ -50,10 +50,10 @@ - + - + @@ -128,7 +128,7 @@ - + @@ -148,7 +148,7 @@ - + @@ -158,7 +158,7 @@ - + @@ -168,7 +168,7 @@ - + @@ -181,14 +181,14 @@ - + - + @@ -254,7 +254,7 @@ - + @@ -264,7 +264,7 @@ - + @@ -287,7 +287,7 @@ - + @@ -297,7 +297,7 @@ - + @@ -307,14 +307,14 @@ - + - + @@ -328,7 +328,7 @@ - + diff --git a/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchSettingsPanel.java b/Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/FileExtMismatchSettingsPanel.java similarity index 99% rename from FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchSettingsPanel.java rename to Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/FileExtMismatchSettingsPanel.java index 98d701cbca..ee76aa4095 100644 --- a/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchSettingsPanel.java +++ b/Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/FileExtMismatchSettingsPanel.java @@ -16,7 +16,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.sleuthkit.autopsy.fileextmismatch; +package org.sleuthkit.autopsy.modules.fileextmismatch; import java.awt.Color; import java.util.ArrayList; @@ -32,7 +32,7 @@ import javax.swing.table.AbstractTableModel; import org.sleuthkit.autopsy.ingest.IngestModuleGlobalSetttingsPanel; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.coreutils.Logger; -import org.sleuthkit.autopsy.filetypeid.FileTypeIdIngestModule; +import org.sleuthkit.autopsy.modules.filetypeid.FileTypeIdIngestModule; import org.sleuthkit.autopsy.corecomponents.OptionsPanel; /** @@ -154,7 +154,7 @@ final class FileExtMismatchSettingsPanel extends IngestModuleGlobalSetttingsPane extRemoveErrLabel = new javax.swing.JLabel(); saveMsgLabel = new javax.swing.JLabel(); - saveButton.setIcon(new javax.swing.ImageIcon(getClass().getResource("/org/sleuthkit/autopsy/fileextmismatch/save16.png"))); // NOI18N + saveButton.setIcon(new javax.swing.ImageIcon(getClass().getResource("/org/sleuthkit/autopsy/modules/fileextmismatch/save16.png"))); // NOI18N saveButton.setText(org.openide.util.NbBundle.getMessage(FileExtMismatchSettingsPanel.class, "FileExtMismatchSettingsPanel.saveButton.text")); // NOI18N saveButton.setEnabled(false); saveButton.addActionListener(new java.awt.event.ActionListener() { diff --git a/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchXML.java b/Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/FileExtMismatchXML.java similarity index 98% rename from FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchXML.java rename to Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/FileExtMismatchXML.java index 8fe9e84a89..d90802a69b 100644 --- a/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchXML.java +++ b/Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/FileExtMismatchXML.java @@ -17,7 +17,7 @@ * limitations under the License. */ -package org.sleuthkit.autopsy.fileextmismatch; +package org.sleuthkit.autopsy.modules.fileextmismatch; import java.io.File; import java.io.IOException; @@ -61,7 +61,7 @@ class FileExtMismatchXML { this.filePath = filePath; try { - boolean extracted = PlatformUtil.extractResourceToUserConfigDir(FileExtMismatchXML.class, DEFAULT_CONFIG_FILE_NAME); + boolean extracted = PlatformUtil.extractResourceToUserConfigDir(FileExtMismatchXML.class, DEFAULT_CONFIG_FILE_NAME, false); } catch (IOException ex) { logger.log(Level.SEVERE, "Error copying default mismatch configuration to user dir ", ex); } diff --git a/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/MismatchConfigSchema.xsd b/Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/MismatchConfigSchema.xsd similarity index 100% rename from FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/MismatchConfigSchema.xsd rename to Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/MismatchConfigSchema.xsd diff --git a/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/mismatch_config.xml b/Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/mismatch_config.xml similarity index 100% rename from FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/mismatch_config.xml rename to Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/mismatch_config.xml diff --git a/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/options-icon.png b/Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/options-icon.png similarity index 100% rename from FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/options-icon.png rename to Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/options-icon.png diff --git a/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/save16.png b/Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/save16.png similarity index 100% rename from FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/save16.png rename to Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/save16.png diff --git a/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/Bundle.properties b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/Bundle.properties similarity index 100% rename from FileTypeId/src/org/sleuthkit/autopsy/filetypeid/Bundle.properties rename to Core/src/org/sleuthkit/autopsy/modules/filetypeid/Bundle.properties diff --git a/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/Bundle_ja.properties b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/Bundle_ja.properties similarity index 100% rename from FileTypeId/src/org/sleuthkit/autopsy/filetypeid/Bundle_ja.properties rename to Core/src/org/sleuthkit/autopsy/modules/filetypeid/Bundle_ja.properties diff --git a/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeDetectionInterface.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeDetectionInterface.java similarity index 96% rename from FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeDetectionInterface.java rename to Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeDetectionInterface.java index 50331c3a88..d748578f91 100644 --- a/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeDetectionInterface.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeDetectionInterface.java @@ -16,7 +16,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.sleuthkit.autopsy.filetypeid; +package org.sleuthkit.autopsy.modules.filetypeid; import org.sleuthkit.datamodel.AbstractFile; diff --git a/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeIdIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java similarity index 68% rename from FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeIdIngestModule.java rename to Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java index 135f6899a9..0ba451f716 100644 --- a/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeIdIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java @@ -16,13 +16,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.sleuthkit.autopsy.filetypeid; +package org.sleuthkit.autopsy.modules.filetypeid; +import java.util.concurrent.atomic.AtomicLong; import java.util.logging.Level; - import org.openide.util.NbBundle; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.ingest.FileIngestModule; +import org.sleuthkit.autopsy.ingest.IngestJobContext; import org.sleuthkit.autopsy.ingest.IngestMessage; import org.sleuthkit.autopsy.ingest.IngestServices; import org.sleuthkit.datamodel.AbstractFile; @@ -34,6 +35,7 @@ import org.sleuthkit.datamodel.TskData.FileKnown; import org.sleuthkit.datamodel.TskException; import org.sleuthkit.autopsy.ingest.IngestModule.ProcessResult; import org.sleuthkit.autopsy.ingest.IngestModuleAdapter; +import org.sleuthkit.autopsy.ingest.IngestModuleReferenceCounter; /** * Detects the type of a file based on signature (magic) values. Posts results @@ -44,9 +46,11 @@ public class FileTypeIdIngestModule extends IngestModuleAdapter implements FileI private static final Logger logger = Logger.getLogger(FileTypeIdIngestModule.class.getName()); private static final long MIN_FILE_SIZE = 512; private final FileTypeIdModuleSettings settings; - private long matchTime = 0; - private int messageId = 0; // RJCTODO: If this is not made a thread safe static, duplicate message ids will be used - private long numFiles = 0; + private long jobId; + private static AtomicLong matchTime = new AtomicLong(0); + private static AtomicLong numFiles = new AtomicLong(0); + private static final IngestModuleReferenceCounter refCounter = new IngestModuleReferenceCounter(); + // The detector. Swap out with a different implementation of FileTypeDetectionInterface as needed. // If desired in the future to be more knowledgable about weird files or rare formats, we could // actually have a list of detectors which are called in order until a match is found. @@ -56,6 +60,12 @@ public class FileTypeIdIngestModule extends IngestModuleAdapter implements FileI this.settings = settings; } + @Override + public void startUp(IngestJobContext context) throws IngestModuleException { + jobId = context.getJobId(); + refCounter.incrementAndGet(jobId); + } + @Override public ProcessResult process(AbstractFile abstractFile) { // skip non-files @@ -76,8 +86,8 @@ public class FileTypeIdIngestModule extends IngestModuleAdapter implements FileI try { long startTime = System.currentTimeMillis(); FileTypeDetectionInterface.FileIdInfo fileId = detector.attemptMatch(abstractFile); - matchTime += (System.currentTimeMillis() - startTime); - numFiles++; + matchTime.getAndAdd(System.currentTimeMillis() - startTime); + numFiles.getAndIncrement(); if (!fileId.type.isEmpty()) { // add artifact @@ -99,22 +109,25 @@ public class FileTypeIdIngestModule extends IngestModuleAdapter implements FileI @Override public void shutDown(boolean ingestJobCancelled) { - StringBuilder detailsSb = new StringBuilder(); - detailsSb.append(""); - detailsSb.append(""); - detailsSb.append("\n"); - detailsSb.append("\n"); - detailsSb.append("
").append(FileTypeIdModuleFactory.getModuleName()).append("
") - .append(NbBundle.getMessage(this.getClass(), "FileTypeIdIngestModule.complete.totalProcTime")) - .append("").append(matchTime).append("
") - .append(NbBundle.getMessage(this.getClass(), "FileTypeIdIngestModule.complete.totalFiles")) - .append("").append(numFiles).append("
"); - IngestServices.getInstance().postMessage(IngestMessage.createMessage(++messageId, IngestMessage.MessageType.INFO, FileTypeIdModuleFactory.getModuleName(), - NbBundle.getMessage(this.getClass(), - "FileTypeIdIngestModule.complete.srvMsg.text"), - detailsSb.toString())); + // We only need to post the summary msg from the last module per job + if (refCounter.decrementAndGet(jobId) == 0) { + StringBuilder detailsSb = new StringBuilder(); + detailsSb.append(""); + detailsSb.append(""); + detailsSb.append("\n"); + detailsSb.append("\n"); + detailsSb.append("
").append(FileTypeIdModuleFactory.getModuleName()).append("
") + .append(NbBundle.getMessage(this.getClass(), "FileTypeIdIngestModule.complete.totalProcTime")) + .append("").append(matchTime.get()).append("
") + .append(NbBundle.getMessage(this.getClass(), "FileTypeIdIngestModule.complete.totalFiles")) + .append("").append(numFiles.get()).append("
"); + IngestServices.getInstance().postMessage(IngestMessage.createMessage(IngestMessage.MessageType.INFO, FileTypeIdModuleFactory.getModuleName(), + NbBundle.getMessage(this.getClass(), + "FileTypeIdIngestModule.complete.srvMsg.text"), + detailsSb.toString())); + } } - + /** * Validate if a given mime type is in the detector's registry. * diff --git a/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeIdModuleFactory.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleFactory.java similarity index 98% rename from FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeIdModuleFactory.java rename to Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleFactory.java index 36004c26e0..0e041f116e 100755 --- a/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeIdModuleFactory.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleFactory.java @@ -16,7 +16,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.sleuthkit.autopsy.filetypeid; +package org.sleuthkit.autopsy.modules.filetypeid; import org.openide.util.NbBundle; import org.openide.util.lookup.ServiceProvider; diff --git a/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeIdModuleSettings.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleSettings.java similarity index 96% rename from FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeIdModuleSettings.java rename to Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleSettings.java index cd2bc4bc12..3dbb50c8a9 100755 --- a/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeIdModuleSettings.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleSettings.java @@ -16,7 +16,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.sleuthkit.autopsy.filetypeid; +package org.sleuthkit.autopsy.modules.filetypeid; import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettings; diff --git a/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeIdModuleSettingsPanel.form b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleSettingsPanel.form similarity index 78% rename from FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeIdModuleSettingsPanel.form rename to Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleSettingsPanel.form index da4b241fb4..6a81b507d5 100644 --- a/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeIdModuleSettingsPanel.form +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleSettingsPanel.form @@ -38,13 +38,13 @@ - + - + - + diff --git a/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeIdModuleSettingsPanel.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleSettingsPanel.java similarity index 98% rename from FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeIdModuleSettingsPanel.java rename to Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleSettingsPanel.java index 442e2b990e..b6f9b75c15 100644 --- a/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeIdModuleSettingsPanel.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleSettingsPanel.java @@ -16,7 +16,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.sleuthkit.autopsy.filetypeid; +package org.sleuthkit.autopsy.modules.filetypeid; import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettings; import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettingsPanel; diff --git a/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/TikaFileTypeDetector.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/TikaFileTypeDetector.java similarity index 95% rename from FileTypeId/src/org/sleuthkit/autopsy/filetypeid/TikaFileTypeDetector.java rename to Core/src/org/sleuthkit/autopsy/modules/filetypeid/TikaFileTypeDetector.java index e074e0f003..06794331c7 100644 --- a/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/TikaFileTypeDetector.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/TikaFileTypeDetector.java @@ -16,7 +16,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.sleuthkit.autopsy.filetypeid; +package org.sleuthkit.autopsy.modules.filetypeid; import java.util.SortedSet; import org.apache.tika.Tika; @@ -35,7 +35,8 @@ class TikaFileTypeDetector implements FileTypeDetectionInterface { FileTypeDetectionInterface.FileIdInfo ret = new FileTypeDetectionInterface.FileIdInfo(); final int maxBytesInitial = 100; //how many bytes to read on first pass byte buffer[] = new byte[maxBytesInitial]; - + int len = abstractFile.read(buffer, 0, maxBytesInitial); + boolean found = false; try { // the xml detection in Tika tries to parse the entire file and throws exceptions @@ -49,7 +50,7 @@ class TikaFileTypeDetector implements FileTypeDetectionInterface { } catch (IndexOutOfBoundsException e) { // do nothing - } + } if (found == false) { String mimetype = tikaInst.detect(buffer); diff --git a/SevenZip/src/org/sleuthkit/autopsy/sevenzip/ArchiveFileExtractorModuleFactory.java b/Core/src/org/sleuthkit/autopsy/modules/sevenzip/ArchiveFileExtractorModuleFactory.java similarity index 97% rename from SevenZip/src/org/sleuthkit/autopsy/sevenzip/ArchiveFileExtractorModuleFactory.java rename to Core/src/org/sleuthkit/autopsy/modules/sevenzip/ArchiveFileExtractorModuleFactory.java index 7bc38efd53..c78eeb2013 100755 --- a/SevenZip/src/org/sleuthkit/autopsy/sevenzip/ArchiveFileExtractorModuleFactory.java +++ b/Core/src/org/sleuthkit/autopsy/modules/sevenzip/ArchiveFileExtractorModuleFactory.java @@ -16,7 +16,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.sleuthkit.autopsy.sevenzip; +package org.sleuthkit.autopsy.modules.sevenzip; import org.openide.util.NbBundle; import org.openide.util.lookup.ServiceProvider; diff --git a/SevenZip/src/org/sleuthkit/autopsy/sevenzip/Bundle.properties b/Core/src/org/sleuthkit/autopsy/modules/sevenzip/Bundle.properties similarity index 100% rename from SevenZip/src/org/sleuthkit/autopsy/sevenzip/Bundle.properties rename to Core/src/org/sleuthkit/autopsy/modules/sevenzip/Bundle.properties diff --git a/SevenZip/src/org/sleuthkit/autopsy/sevenzip/Bundle_ja.properties b/Core/src/org/sleuthkit/autopsy/modules/sevenzip/Bundle_ja.properties similarity index 100% rename from SevenZip/src/org/sleuthkit/autopsy/sevenzip/Bundle_ja.properties rename to Core/src/org/sleuthkit/autopsy/modules/sevenzip/Bundle_ja.properties diff --git a/SevenZip/src/org/sleuthkit/autopsy/sevenzip/SevenZipContentReadStream.java b/Core/src/org/sleuthkit/autopsy/modules/sevenzip/SevenZipContentReadStream.java similarity index 98% rename from SevenZip/src/org/sleuthkit/autopsy/sevenzip/SevenZipContentReadStream.java rename to Core/src/org/sleuthkit/autopsy/modules/sevenzip/SevenZipContentReadStream.java index 9e180e7829..2b7cd375c9 100644 --- a/SevenZip/src/org/sleuthkit/autopsy/sevenzip/SevenZipContentReadStream.java +++ b/Core/src/org/sleuthkit/autopsy/modules/sevenzip/SevenZipContentReadStream.java @@ -16,7 +16,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.sleuthkit.autopsy.sevenzip; +package org.sleuthkit.autopsy.modules.sevenzip; import java.io.IOException; import java.util.logging.Level; diff --git a/SevenZip/src/org/sleuthkit/autopsy/sevenzip/SevenZipIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/sevenzip/SevenZipIngestModule.java similarity index 94% rename from SevenZip/src/org/sleuthkit/autopsy/sevenzip/SevenZipIngestModule.java rename to Core/src/org/sleuthkit/autopsy/modules/sevenzip/SevenZipIngestModule.java index b5778984bf..da517d7b39 100644 --- a/SevenZip/src/org/sleuthkit/autopsy/sevenzip/SevenZipIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/sevenzip/SevenZipIngestModule.java @@ -16,7 +16,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.sleuthkit.autopsy.sevenzip; +package org.sleuthkit.autopsy.modules.sevenzip; import java.io.BufferedOutputStream; import java.io.File; @@ -60,6 +60,7 @@ import org.sleuthkit.autopsy.ingest.IngestModule.ProcessResult; import org.sleuthkit.autopsy.ingest.IngestJobContext; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.ingest.ModuleDataEvent; +import org.sleuthkit.autopsy.ingest.IngestModuleReferenceCounter; /** * 7Zip ingest module extracts supported archives, adds extracted DerivedFiles, @@ -69,7 +70,6 @@ public final class SevenZipIngestModule extends IngestModuleAdapter implements F private static final Logger logger = Logger.getLogger(SevenZipIngestModule.class.getName()); private IngestServices services = IngestServices.getInstance(); - private volatile int messageID = 0; // RJCTODO: This is not actually thread safe static final String[] SUPPORTED_EXTENSIONS = {"zip", "rar", "arj", "7z", "7zip", "gzip", "gz", "bzip2", "tar", "tgz",}; // "iso"}; private String unpackDir; //relative to the case, to store in db private String unpackDirPath; //absolute, to extract to @@ -91,6 +91,8 @@ public final class SevenZipIngestModule extends IngestModuleAdapter implements F private final byte[] fileHeaderBuffer = new byte[readHeaderSize]; private static final int ZIP_SIGNATURE_BE = 0x504B0304; private IngestJobContext context; + private long jobId; + private final static IngestModuleReferenceCounter refCounter = new IngestModuleReferenceCounter(); SevenZipIngestModule() { } @@ -98,6 +100,7 @@ public final class SevenZipIngestModule extends IngestModuleAdapter implements F @Override public void startUp(IngestJobContext context) throws IngestModuleException { this.context = context; + jobId = context.getJobId(); final Case currentCase = Case.getCurrentCase(); @@ -117,25 +120,28 @@ public final class SevenZipIngestModule extends IngestModuleAdapter implements F String details = NbBundle.getMessage(this.getClass(), "SevenZipIngestModule.init.errInitModule.details", unpackDirPath, e.getMessage()); - services.postMessage(IngestMessage.createErrorMessage(++messageID, ArchiveFileExtractorModuleFactory.getModuleName(), msg, details)); + services.postMessage(IngestMessage.createErrorMessage(ArchiveFileExtractorModuleFactory.getModuleName(), msg, details)); throw e; } } - try { - SevenZip.initSevenZipFromPlatformJAR(); - String platform = SevenZip.getUsedPlatform(); - logger.log(Level.INFO, "7-Zip-JBinding library was initialized on supported platform: {0}", platform); - } catch (SevenZipNativeInitializationException e) { - logger.log(Level.SEVERE, "Error initializing 7-Zip-JBinding library", e); - String msg = NbBundle.getMessage(this.getClass(), "SevenZipIngestModule.init.errInitModule.msg", - ArchiveFileExtractorModuleFactory.getModuleName()); - String details = NbBundle.getMessage(this.getClass(), "SevenZipIngestModule.init.errCantInitLib", - e.getMessage()); - services.postMessage(IngestMessage.createErrorMessage(++messageID, ArchiveFileExtractorModuleFactory.getModuleName(), msg, details)); - throw new RuntimeException(e); + // if first instance of this module for this job then check 7zip init + if (refCounter.incrementAndGet(jobId) == 1) { + try { + SevenZip.initSevenZipFromPlatformJAR(); + String platform = SevenZip.getUsedPlatform(); + logger.log(Level.INFO, "7-Zip-JBinding library was initialized on supported platform: {0}", platform); + } catch (SevenZipNativeInitializationException e) { + logger.log(Level.SEVERE, "Error initializing 7-Zip-JBinding library", e); + String msg = NbBundle.getMessage(this.getClass(), "SevenZipIngestModule.init.errInitModule.msg", + ArchiveFileExtractorModuleFactory.getModuleName()); + String details = NbBundle.getMessage(this.getClass(), "SevenZipIngestModule.init.errCantInitLib", + e.getMessage()); + services.postMessage(IngestMessage.createErrorMessage(ArchiveFileExtractorModuleFactory.getModuleName(), msg, details)); + throw new RuntimeException(e); + } } - + archiveDepthCountTree = new ArchiveDepthCountTree(); } @@ -180,6 +186,12 @@ public final class SevenZipIngestModule extends IngestModuleAdapter implements F return ProcessResult.OK; } + @Override + public void shutDown(boolean ingestJobCancelled) { + // We don't need the value, but for cleanliness and consistency, -- it + refCounter.decrementAndGet(jobId); + } + private void sendNewFilesEvent(AbstractFile archive, List unpackedFiles) { //currently sending a single event for all new files services.fireModuleContentEvent(new ModuleContentEvent(archive)); @@ -242,7 +254,7 @@ public final class SevenZipIngestModule extends IngestModuleAdapter implements F String details = NbBundle.getMessage(this.getClass(), "SevenZipIngestModule.isZipBombCheck.warnDetails", cRatio); //MessageNotifyUtil.Notify.error(msg, details); - services.postMessage(IngestMessage.createWarningMessage(++messageID, ArchiveFileExtractorModuleFactory.getModuleName(), msg, details)); + services.postMessage(IngestMessage.createWarningMessage(ArchiveFileExtractorModuleFactory.getModuleName(), msg, details)); return true; } else { return false; @@ -276,7 +288,7 @@ public final class SevenZipIngestModule extends IngestModuleAdapter implements F "SevenZipIngestModule.unpack.warnDetails.zipBomb", parentAr.getDepth()); //MessageNotifyUtil.Notify.error(msg, details); - services.postMessage(IngestMessage.createWarningMessage(++messageID, ArchiveFileExtractorModuleFactory.getModuleName(), msg, details)); + services.postMessage(IngestMessage.createWarningMessage(ArchiveFileExtractorModuleFactory.getModuleName(), msg, details)); return unpackedFiles; } @@ -403,7 +415,7 @@ public final class SevenZipIngestModule extends IngestModuleAdapter implements F String details = NbBundle.getMessage(this.getClass(), "SevenZipIngestModule.unpack.notEnoughDiskSpace.details"); //MessageNotifyUtil.Notify.error(msg, details); - services.postMessage(IngestMessage.createErrorMessage(++messageID, ArchiveFileExtractorModuleFactory.getModuleName(), msg, details)); + services.postMessage(IngestMessage.createErrorMessage(ArchiveFileExtractorModuleFactory.getModuleName(), msg, details)); logger.log(Level.INFO, "Skipping archive item due not sufficient disk space for this item: {0}, {1}", new Object[]{archiveFile.getName(), fileName}); continue; //skip this file } else { @@ -501,7 +513,7 @@ public final class SevenZipIngestModule extends IngestModuleAdapter implements F String details = NbBundle.getMessage(this.getClass(), "SevenZipIngestModule.unpack.errUnpacking.details", fullName, ex.getMessage()); - services.postMessage(IngestMessage.createErrorMessage(++messageID, ArchiveFileExtractorModuleFactory.getModuleName(), msg, details)); + services.postMessage(IngestMessage.createErrorMessage(ArchiveFileExtractorModuleFactory.getModuleName(), msg, details)); } } finally { if (inArchive != null) { @@ -541,7 +553,7 @@ public final class SevenZipIngestModule extends IngestModuleAdapter implements F String details = NbBundle.getMessage(this.getClass(), "SevenZipIngestModule.unpack.encrFileDetected.details", archiveFile.getName(), ArchiveFileExtractorModuleFactory.getModuleName()); - services.postMessage(IngestMessage.createWarningMessage(++messageID, ArchiveFileExtractorModuleFactory.getModuleName(), msg, details)); + services.postMessage(IngestMessage.createWarningMessage(ArchiveFileExtractorModuleFactory.getModuleName(), msg, details)); } return unpackedFiles; diff --git a/Core/src/org/sleuthkit/autopsy/report/Bundle.properties b/Core/src/org/sleuthkit/autopsy/report/Bundle.properties index 5ef11a4a2a..f3d0b5dbb2 100644 --- a/Core/src/org/sleuthkit/autopsy/report/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/report/Bundle.properties @@ -47,6 +47,13 @@ ReportBodyFile.progress.processing=Now processing {0}... ReportBodyFile.getName.text=TSK Body File ReportBodyFile.getDesc.text=Body file format report with MAC times for every file. This format can be used for a timeline view. ReportBodyFile.getFilePath.text=BodyFile.txt +ReportKML.progress.querying=Querying files... +ReportKML.ingestWarning.text=Warning, this report was run before ingest services completed\! +ReportKML.progress.loading=Loading files... +ReportKML.progress.processing=Now processing {0}... +ReportKML.getName.text=Google Earth/KML +ReportKML.getDesc.text=KML format report with coordinates for relevant files. This format can be used for google earth views. +ReportKML.getFilePath.text=ReportKML.kml ReportBranding.defaultReportTitle.text=Autopsy Forensic Report ReportBranding.defaultReportFooter.text=Powered by Autopsy Open Source Digital Forensics Platform - www.sleuthkit.org ReportExcel.numAartifacts.text=Number of artifacts\: diff --git a/Core/src/org/sleuthkit/autopsy/report/ReportKML.java b/Core/src/org/sleuthkit/autopsy/report/ReportKML.java new file mode 100644 index 0000000000..05cbb3b6a7 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/report/ReportKML.java @@ -0,0 +1,318 @@ +/* + * + * Autopsy Forensic Browser + * + * Copyright 2014 Basis Technology Corp. + * contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.report; + +import javax.swing.JPanel; + +import org.openide.util.NbBundle; +import org.sleuthkit.autopsy.casemodule.Case; +import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.datamodel.*; +import org.sleuthkit.autopsy.ingest.IngestManager; +import org.sleuthkit.datamodel.BlackboardArtifact; +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileOutputStream; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.logging.Level; +import org.jdom2.Document; +import org.jdom2.Element; +import org.jdom2.Namespace; +import org.jdom2.output.Format; +import org.jdom2.output.XMLOutputter; +import org.apache.commons.lang.StringEscapeUtils; + +/** + * Generates a KML file based on geo coordinates store in blackboard. + */ +class ReportKML implements GeneralReportModule { + + private static final Logger logger = Logger.getLogger(ReportKML.class.getName()); + private static ReportKML instance = null; + private Case currentCase; + private SleuthkitCase skCase; + private String reportPath; + + // Hidden constructor for the report + private ReportKML() { + } + + // Get the default implementation of this report + public static synchronized ReportKML getDefault() { + if (instance == null) { + instance = new ReportKML(); + } + return instance; + } + + /** + * Generates a body file format report for use with the MAC time tool. + * + * @param path path to save the report + * @param progressPanel panel to update the report's progress + */ + @Override + public void generateReport(String path, ReportProgressPanel progressPanel) { + + // Start the progress bar and setup the report + progressPanel.setIndeterminate(false); + progressPanel.start(); + progressPanel.updateStatusLabel(NbBundle.getMessage(this.getClass(), "ReportKML.progress.querying")); + reportPath = path + "ReportKML.kml"; + String reportPath2 = path + "ReportKML.txt"; + currentCase = Case.getCurrentCase(); + skCase = currentCase.getSleuthkitCase(); + + progressPanel.updateStatusLabel(NbBundle.getMessage(this.getClass(), "ReportKML.progress.loading")); + // Check if ingest has finished + String ingestwarning = ""; + if (IngestManager.getInstance().isIngestRunning()) { + ingestwarning = NbBundle.getMessage(this.getClass(), "ReportBodyFile.ingestWarning.text"); + } + progressPanel.setMaximumProgress(5); + progressPanel.increment(); + + + try { + + BufferedWriter out = null; + try { + out = new BufferedWriter(new FileWriter(reportPath2)); + + double lat = 0; // temp latitude + double lon = 0; //temp longitude + AbstractFile aFile; + String geoPath = ""; // will hold values of images to put in kml + String imageName = ""; + + + File f; + for (BlackboardArtifact artifact : skCase.getBlackboardArtifacts(BlackboardArtifact.ARTIFACT_TYPE.TSK_METADATA_EXIF)) { + lat = 0; + lon = 0; + geoPath = ""; + String extractedToPath; + for (BlackboardAttribute attribute : artifact.getAttributes()) { + if (attribute.getAttributeTypeID() == BlackboardAttribute.ATTRIBUTE_TYPE.TSK_GEO_LATITUDE.getTypeID()) //latitude + { + + lat = attribute.getValueDouble(); + } + if (attribute.getAttributeTypeID() == BlackboardAttribute.ATTRIBUTE_TYPE.TSK_GEO_LONGITUDE.getTypeID()) //longitude + { + lon = attribute.getValueDouble(); + } + } + if (lon != 0 && lat != 0) { + aFile = artifact.getSleuthkitCase().getAbstractFileById(artifact.getObjectID()); + + extractedToPath = reportPath + aFile.getName(); + geoPath = extractedToPath; + f = new File(extractedToPath); + f.createNewFile(); + copyFileUsingStream(aFile, f); + imageName = aFile.getName(); + out.write(String.valueOf(lat)); + out.write(";"); + out.write(String.valueOf(lon)); + out.write(";"); + out.write(String.valueOf(geoPath)); + out.write(";"); + out.write(String.valueOf(imageName)); + out.write("\n"); + // lat lon path name + } + } + out.flush(); + out.close(); + progressPanel.increment(); + /* + * Step 1: generate XML stub + */ + Namespace ns = Namespace.getNamespace("", "http://earth.google.com/kml/2.2"); + // kml + Element kml = new Element("kml", ns); + Document kmlDocument = new Document(kml); + + // Document + Element document = new Element("Document", ns); + kml.addContent(document); + + // name + Element name = new Element("name", ns); + name.setText("Java Generated KML Document"); + document.addContent(name); + + /* + * Step 2: add in Style elements + */ + + // Style + Element style = new Element("Style", ns); + style.setAttribute("id", "redIcon"); + document.addContent(style); + + // IconStyle + Element iconStyle = new Element("IconStyle", ns); + style.addContent(iconStyle); + + // color + Element color = new Element("color", ns); + color.setText("990000ff"); + iconStyle.addContent(color); + + // Icon + Element icon = new Element("Icon", ns); + iconStyle.addContent(icon); + + // href + Element href = new Element("href", ns); + href.setText("http://www.cs.mun.ca/~hoeber/teaching/cs4767/notes/02.1-kml/circle.png"); + icon.addContent(href); + progressPanel.increment(); + /* + * Step 3: read data from source location and + * add in a Placemark for each data element + */ + + File file = new File(reportPath2); + BufferedReader reader; + + reader = new BufferedReader(new FileReader(file)); + + String line = reader.readLine(); + while (line != null) { + String[] lineParts = line.split(";"); + if (lineParts.length == 4) { + String coordinates = lineParts[1].trim() + "," + lineParts[0].trim(); //lat,lon + // Placemark + Element placemark = new Element("Placemark", ns); + document.addContent(placemark); + + // name + Element pmName = new Element("name", ns); + pmName.setText(lineParts[3].trim()); + placemark.addContent(pmName); + + // Path + Element pmPath = new Element("Path", ns); + pmPath.setText(lineParts[2].trim()); + placemark.addContent(pmPath); + + // description + Element pmDescription = new Element("description", ns); + String xml = "
- - - - - Builds, tests, and runs the project org.sleuthkit.autopsy.exifextract. - - \ No newline at end of file diff --git a/ExifParser/manifest.mf b/ExifParser/manifest.mf deleted file mode 100644 index 40668bc3a3..0000000000 --- a/ExifParser/manifest.mf +++ /dev/null @@ -1,6 +0,0 @@ -Manifest-Version: 1.0 -AutoUpdate-Show-In-Client: true -OpenIDE-Module: org.sleuthkit.autopsy.exifparser/3 -OpenIDE-Module-Implementation-Version: 10 -OpenIDE-Module-Layer: org/sleuthkit/autopsy/exifparser/layer.xml -OpenIDE-Module-Localizing-Bundle: org/sleuthkit/autopsy/exifparser/Bundle.properties diff --git a/ExifParser/nbproject/build-impl.xml b/ExifParser/nbproject/build-impl.xml deleted file mode 100644 index e1bcfa276c..0000000000 --- a/ExifParser/nbproject/build-impl.xml +++ /dev/null @@ -1,45 +0,0 @@ - - - - - - - - - - - - - You must set 'suite.dir' to point to your containing module suite - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/ExifParser/nbproject/project.properties b/ExifParser/nbproject/project.properties deleted file mode 100644 index f278b6de1b..0000000000 --- a/ExifParser/nbproject/project.properties +++ /dev/null @@ -1,6 +0,0 @@ -javac.source=1.7 -javac.compilerargs=-Xlint -Xlint:-serial -license.file=../LICENSE-2.0.txt -nbm.homepage=http://www.sleuthkit.org/autopsy/ -nbm.needs.restart=true -spec.version.base=1.1 diff --git a/ExifParser/nbproject/project.xml b/ExifParser/nbproject/project.xml deleted file mode 100644 index 88405985e9..0000000000 --- a/ExifParser/nbproject/project.xml +++ /dev/null @@ -1,46 +0,0 @@ - - - org.netbeans.modules.apisupport.project - - - org.sleuthkit.autopsy.exifparser - - - - org.openide.util - - - - 8.29.3 - - - - org.openide.util.lookup - - - - 8.19.1 - - - - org.sleuthkit.autopsy.core - - - - 9 - 7.1 - - - - - - ext/xmpcore.jar - release/modules/ext/xmpcore.jar - - - ext/metadata-extractor-2.6.2.jar - release/modules/ext/metadata-extractor-2.6.2.jar - - - - diff --git a/ExifParser/nbproject/suite.properties b/ExifParser/nbproject/suite.properties deleted file mode 100644 index 29d7cc9bd6..0000000000 --- a/ExifParser/nbproject/suite.properties +++ /dev/null @@ -1 +0,0 @@ -suite.dir=${basedir}/.. diff --git a/FileExtMismatch/build.xml b/FileExtMismatch/build.xml deleted file mode 100644 index 90f199456e..0000000000 --- a/FileExtMismatch/build.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - Builds, tests, and runs the project org.sleuthkit.autopsy.fileextmismatch. - - diff --git a/FileExtMismatch/manifest.mf b/FileExtMismatch/manifest.mf deleted file mode 100644 index c1678923ea..0000000000 --- a/FileExtMismatch/manifest.mf +++ /dev/null @@ -1,5 +0,0 @@ -Manifest-Version: 1.0 -OpenIDE-Module: org.sleuthkit.autopsy.fileextmismatch/3 -OpenIDE-Module-Implementation-Version: 1 -OpenIDE-Module-Localizing-Bundle: org/sleuthkit/autopsy/fileextmismatch/Bundle.properties - diff --git a/FileExtMismatch/nbproject/build-impl.xml b/FileExtMismatch/nbproject/build-impl.xml deleted file mode 100644 index 40c37cb50f..0000000000 --- a/FileExtMismatch/nbproject/build-impl.xml +++ /dev/null @@ -1,45 +0,0 @@ - - - - - - - - - - - - - You must set 'suite.dir' to point to your containing module suite - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/FileExtMismatch/nbproject/platform.properties b/FileExtMismatch/nbproject/platform.properties deleted file mode 100644 index a9fa87f749..0000000000 --- a/FileExtMismatch/nbproject/platform.properties +++ /dev/null @@ -1,120 +0,0 @@ -branding.token=autopsy -netbeans-plat-version=7.3.1 -suite.dir=${basedir} -nbplatform.active.dir=${suite.dir}/netbeans-plat/${netbeans-plat-version} -harness.dir=${nbplatform.active.dir}/harness -bootstrap.url=http://deadlock.netbeans.org/hudson/job/nbms-and-javadoc/lastStableBuild/artifact/nbbuild/netbeans/harness/tasks.jar -autoupdate.catalog.url=http://dlc.sun.com.edgesuite.net/netbeans/updates/${netbeans-plat-version}/uc/final/distribution/catalog.xml.gz -cluster.path=\ - ${nbplatform.active.dir}/harness:\ - ${nbplatform.active.dir}/java:\ - ${nbplatform.active.dir}/platform -disabled.modules=\ - org.apache.tools.ant.module,\ - org.netbeans.api.debugger.jpda,\ - org.netbeans.api.java,\ - org.netbeans.lib.nbjavac,\ - org.netbeans.libs.cglib,\ - org.netbeans.libs.javacapi,\ - org.netbeans.libs.javacimpl,\ - org.netbeans.libs.springframework,\ - org.netbeans.modules.ant.browsetask,\ - org.netbeans.modules.ant.debugger,\ - org.netbeans.modules.ant.freeform,\ - org.netbeans.modules.ant.grammar,\ - org.netbeans.modules.ant.kit,\ - org.netbeans.modules.beans,\ - org.netbeans.modules.classfile,\ - org.netbeans.modules.dbschema,\ - org.netbeans.modules.debugger.jpda,\ - org.netbeans.modules.debugger.jpda.ant,\ - org.netbeans.modules.debugger.jpda.kit,\ - org.netbeans.modules.debugger.jpda.projects,\ - org.netbeans.modules.debugger.jpda.ui,\ - org.netbeans.modules.debugger.jpda.visual,\ - org.netbeans.modules.findbugs.installer,\ - org.netbeans.modules.form,\ - org.netbeans.modules.form.binding,\ - org.netbeans.modules.form.j2ee,\ - org.netbeans.modules.form.kit,\ - org.netbeans.modules.form.nb,\ - org.netbeans.modules.form.refactoring,\ - org.netbeans.modules.hibernate,\ - org.netbeans.modules.hibernatelib,\ - org.netbeans.modules.hudson.ant,\ - org.netbeans.modules.hudson.maven,\ - org.netbeans.modules.i18n,\ - org.netbeans.modules.i18n.form,\ - org.netbeans.modules.j2ee.core.utilities,\ - org.netbeans.modules.j2ee.eclipselink,\ - org.netbeans.modules.j2ee.eclipselinkmodelgen,\ - org.netbeans.modules.j2ee.jpa.refactoring,\ - org.netbeans.modules.j2ee.jpa.verification,\ - org.netbeans.modules.j2ee.metadata,\ - org.netbeans.modules.j2ee.metadata.model.support,\ - org.netbeans.modules.j2ee.persistence,\ - org.netbeans.modules.j2ee.persistence.kit,\ - org.netbeans.modules.j2ee.persistenceapi,\ - org.netbeans.modules.java.api.common,\ - org.netbeans.modules.java.debug,\ - org.netbeans.modules.java.editor,\ - org.netbeans.modules.java.editor.lib,\ - org.netbeans.modules.java.examples,\ - org.netbeans.modules.java.freeform,\ - org.netbeans.modules.java.guards,\ - org.netbeans.modules.java.helpset,\ - org.netbeans.modules.java.hints,\ - org.netbeans.modules.java.hints.declarative,\ - org.netbeans.modules.java.hints.declarative.test,\ - org.netbeans.modules.java.hints.legacy.spi,\ - org.netbeans.modules.java.hints.test,\ - org.netbeans.modules.java.hints.ui,\ - org.netbeans.modules.java.j2seplatform,\ - org.netbeans.modules.java.j2seproject,\ - org.netbeans.modules.java.kit,\ - org.netbeans.modules.java.lexer,\ - org.netbeans.modules.java.navigation,\ - org.netbeans.modules.java.platform,\ - org.netbeans.modules.java.preprocessorbridge,\ - org.netbeans.modules.java.project,\ - org.netbeans.modules.java.source,\ - org.netbeans.modules.java.source.ant,\ - org.netbeans.modules.java.source.queries,\ - org.netbeans.modules.java.source.queriesimpl,\ - org.netbeans.modules.java.sourceui,\ - org.netbeans.modules.java.testrunner,\ - org.netbeans.modules.javadoc,\ - org.netbeans.modules.javawebstart,\ - org.netbeans.modules.junit,\ - org.netbeans.modules.maven,\ - org.netbeans.modules.maven.checkstyle,\ - org.netbeans.modules.maven.coverage,\ - org.netbeans.modules.maven.embedder,\ - org.netbeans.modules.maven.grammar,\ - org.netbeans.modules.maven.graph,\ - org.netbeans.modules.maven.hints,\ - org.netbeans.modules.maven.indexer,\ - org.netbeans.modules.maven.junit,\ - org.netbeans.modules.maven.kit,\ - org.netbeans.modules.maven.model,\ - org.netbeans.modules.maven.osgi,\ - org.netbeans.modules.maven.persistence,\ - org.netbeans.modules.maven.refactoring,\ - org.netbeans.modules.maven.repository,\ - org.netbeans.modules.maven.search,\ - org.netbeans.modules.maven.spring,\ - org.netbeans.modules.projectimport.eclipse.core,\ - org.netbeans.modules.projectimport.eclipse.j2se,\ - org.netbeans.modules.refactoring.java,\ - org.netbeans.modules.spellchecker.bindings.java,\ - org.netbeans.modules.spring.beans,\ - org.netbeans.modules.testng,\ - org.netbeans.modules.testng.ant,\ - org.netbeans.modules.testng.maven,\ - org.netbeans.modules.websvc.jaxws21,\ - org.netbeans.modules.websvc.jaxws21api,\ - org.netbeans.modules.websvc.saas.codegen.java,\ - org.netbeans.modules.xml.jaxb,\ - org.netbeans.modules.xml.tools.java,\ - org.netbeans.spi.java.hints - diff --git a/FileExtMismatch/nbproject/project.properties b/FileExtMismatch/nbproject/project.properties deleted file mode 100644 index 21d3761899..0000000000 --- a/FileExtMismatch/nbproject/project.properties +++ /dev/null @@ -1,3 +0,0 @@ -javac.source=1.7 -javac.compilerargs=-Xlint -Xlint:-serial -spec.version.base=1.3 diff --git a/FileExtMismatch/nbproject/project.xml b/FileExtMismatch/nbproject/project.xml deleted file mode 100644 index 23fa0b7529..0000000000 --- a/FileExtMismatch/nbproject/project.xml +++ /dev/null @@ -1,115 +0,0 @@ - - - org.netbeans.modules.apisupport.project - - - org.sleuthkit.autopsy.fileextmismatch - - - - org.netbeans.api.progress - - - - 1 - 1.32.1 - - - - org.netbeans.modules.options.api - - - - 1 - 1.31.2 - - - - org.openide.awt - - - - 7.55.1 - - - - org.openide.dialogs - - - - 7.28.1 - - - - org.openide.explorer - - - - 6.50.3 - - - - org.openide.nodes - - - - 7.33.2 - - - - org.openide.util - - - - 8.29.3 - - - - org.openide.util.lookup - - - - 8.19.1 - - - - org.openide.windows - - - - 6.60.1 - - - - org.sleuthkit.autopsy.core - - - - 9 - 7.0 - - - - org.sleuthkit.autopsy.corelibs - - - - 3 - 1.1 - - - - org.sleuthkit.autopsy.filetypeid - - - - 1.0 - - - - - org.sleuthkit.autopsy.fileextmismatch - - - - diff --git a/FileExtMismatch/nbproject/suite.properties b/FileExtMismatch/nbproject/suite.properties deleted file mode 100644 index 29d7cc9bd6..0000000000 --- a/FileExtMismatch/nbproject/suite.properties +++ /dev/null @@ -1 +0,0 @@ -suite.dir=${basedir}/.. diff --git a/FileTypeId/build.xml b/FileTypeId/build.xml deleted file mode 100644 index 9771a82a0b..0000000000 --- a/FileTypeId/build.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - Builds, tests, and runs the project org.sleuthkit.autopsy.filetypeid. - - diff --git a/FileTypeId/manifest.mf b/FileTypeId/manifest.mf deleted file mode 100644 index eac3c30420..0000000000 --- a/FileTypeId/manifest.mf +++ /dev/null @@ -1,5 +0,0 @@ -Manifest-Version: 1.0 -OpenIDE-Module: org.sleuthkit.autopsy.filetypeid -OpenIDE-Module-Localizing-Bundle: org/sleuthkit/autopsy/filetypeid/Bundle.properties -OpenIDE-Module-Specification-Version: 1.0 - diff --git a/FileTypeId/nbproject/build-impl.xml b/FileTypeId/nbproject/build-impl.xml deleted file mode 100644 index ec094e8ceb..0000000000 --- a/FileTypeId/nbproject/build-impl.xml +++ /dev/null @@ -1,45 +0,0 @@ - - - - - - - - - - - - - You must set 'suite.dir' to point to your containing module suite - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/FileTypeId/nbproject/platform.properties b/FileTypeId/nbproject/platform.properties deleted file mode 100644 index a9fa87f749..0000000000 --- a/FileTypeId/nbproject/platform.properties +++ /dev/null @@ -1,120 +0,0 @@ -branding.token=autopsy -netbeans-plat-version=7.3.1 -suite.dir=${basedir} -nbplatform.active.dir=${suite.dir}/netbeans-plat/${netbeans-plat-version} -harness.dir=${nbplatform.active.dir}/harness -bootstrap.url=http://deadlock.netbeans.org/hudson/job/nbms-and-javadoc/lastStableBuild/artifact/nbbuild/netbeans/harness/tasks.jar -autoupdate.catalog.url=http://dlc.sun.com.edgesuite.net/netbeans/updates/${netbeans-plat-version}/uc/final/distribution/catalog.xml.gz -cluster.path=\ - ${nbplatform.active.dir}/harness:\ - ${nbplatform.active.dir}/java:\ - ${nbplatform.active.dir}/platform -disabled.modules=\ - org.apache.tools.ant.module,\ - org.netbeans.api.debugger.jpda,\ - org.netbeans.api.java,\ - org.netbeans.lib.nbjavac,\ - org.netbeans.libs.cglib,\ - org.netbeans.libs.javacapi,\ - org.netbeans.libs.javacimpl,\ - org.netbeans.libs.springframework,\ - org.netbeans.modules.ant.browsetask,\ - org.netbeans.modules.ant.debugger,\ - org.netbeans.modules.ant.freeform,\ - org.netbeans.modules.ant.grammar,\ - org.netbeans.modules.ant.kit,\ - org.netbeans.modules.beans,\ - org.netbeans.modules.classfile,\ - org.netbeans.modules.dbschema,\ - org.netbeans.modules.debugger.jpda,\ - org.netbeans.modules.debugger.jpda.ant,\ - org.netbeans.modules.debugger.jpda.kit,\ - org.netbeans.modules.debugger.jpda.projects,\ - org.netbeans.modules.debugger.jpda.ui,\ - org.netbeans.modules.debugger.jpda.visual,\ - org.netbeans.modules.findbugs.installer,\ - org.netbeans.modules.form,\ - org.netbeans.modules.form.binding,\ - org.netbeans.modules.form.j2ee,\ - org.netbeans.modules.form.kit,\ - org.netbeans.modules.form.nb,\ - org.netbeans.modules.form.refactoring,\ - org.netbeans.modules.hibernate,\ - org.netbeans.modules.hibernatelib,\ - org.netbeans.modules.hudson.ant,\ - org.netbeans.modules.hudson.maven,\ - org.netbeans.modules.i18n,\ - org.netbeans.modules.i18n.form,\ - org.netbeans.modules.j2ee.core.utilities,\ - org.netbeans.modules.j2ee.eclipselink,\ - org.netbeans.modules.j2ee.eclipselinkmodelgen,\ - org.netbeans.modules.j2ee.jpa.refactoring,\ - org.netbeans.modules.j2ee.jpa.verification,\ - org.netbeans.modules.j2ee.metadata,\ - org.netbeans.modules.j2ee.metadata.model.support,\ - org.netbeans.modules.j2ee.persistence,\ - org.netbeans.modules.j2ee.persistence.kit,\ - org.netbeans.modules.j2ee.persistenceapi,\ - org.netbeans.modules.java.api.common,\ - org.netbeans.modules.java.debug,\ - org.netbeans.modules.java.editor,\ - org.netbeans.modules.java.editor.lib,\ - org.netbeans.modules.java.examples,\ - org.netbeans.modules.java.freeform,\ - org.netbeans.modules.java.guards,\ - org.netbeans.modules.java.helpset,\ - org.netbeans.modules.java.hints,\ - org.netbeans.modules.java.hints.declarative,\ - org.netbeans.modules.java.hints.declarative.test,\ - org.netbeans.modules.java.hints.legacy.spi,\ - org.netbeans.modules.java.hints.test,\ - org.netbeans.modules.java.hints.ui,\ - org.netbeans.modules.java.j2seplatform,\ - org.netbeans.modules.java.j2seproject,\ - org.netbeans.modules.java.kit,\ - org.netbeans.modules.java.lexer,\ - org.netbeans.modules.java.navigation,\ - org.netbeans.modules.java.platform,\ - org.netbeans.modules.java.preprocessorbridge,\ - org.netbeans.modules.java.project,\ - org.netbeans.modules.java.source,\ - org.netbeans.modules.java.source.ant,\ - org.netbeans.modules.java.source.queries,\ - org.netbeans.modules.java.source.queriesimpl,\ - org.netbeans.modules.java.sourceui,\ - org.netbeans.modules.java.testrunner,\ - org.netbeans.modules.javadoc,\ - org.netbeans.modules.javawebstart,\ - org.netbeans.modules.junit,\ - org.netbeans.modules.maven,\ - org.netbeans.modules.maven.checkstyle,\ - org.netbeans.modules.maven.coverage,\ - org.netbeans.modules.maven.embedder,\ - org.netbeans.modules.maven.grammar,\ - org.netbeans.modules.maven.graph,\ - org.netbeans.modules.maven.hints,\ - org.netbeans.modules.maven.indexer,\ - org.netbeans.modules.maven.junit,\ - org.netbeans.modules.maven.kit,\ - org.netbeans.modules.maven.model,\ - org.netbeans.modules.maven.osgi,\ - org.netbeans.modules.maven.persistence,\ - org.netbeans.modules.maven.refactoring,\ - org.netbeans.modules.maven.repository,\ - org.netbeans.modules.maven.search,\ - org.netbeans.modules.maven.spring,\ - org.netbeans.modules.projectimport.eclipse.core,\ - org.netbeans.modules.projectimport.eclipse.j2se,\ - org.netbeans.modules.refactoring.java,\ - org.netbeans.modules.spellchecker.bindings.java,\ - org.netbeans.modules.spring.beans,\ - org.netbeans.modules.testng,\ - org.netbeans.modules.testng.ant,\ - org.netbeans.modules.testng.maven,\ - org.netbeans.modules.websvc.jaxws21,\ - org.netbeans.modules.websvc.jaxws21api,\ - org.netbeans.modules.websvc.saas.codegen.java,\ - org.netbeans.modules.xml.jaxb,\ - org.netbeans.modules.xml.tools.java,\ - org.netbeans.spi.java.hints - diff --git a/FileTypeId/nbproject/project.properties b/FileTypeId/nbproject/project.properties deleted file mode 100644 index 2dfc44febe..0000000000 --- a/FileTypeId/nbproject/project.properties +++ /dev/null @@ -1,3 +0,0 @@ -file.reference.tika-core-1.2.jar=release/modules/ext/tika-core-1.2.jar -javac.source=1.7 -javac.compilerargs=-Xlint -Xlint:-serial diff --git a/FileTypeId/nbproject/project.xml b/FileTypeId/nbproject/project.xml deleted file mode 100644 index 8d3ea957d3..0000000000 --- a/FileTypeId/nbproject/project.xml +++ /dev/null @@ -1,103 +0,0 @@ - - - org.netbeans.modules.apisupport.project - - - org.sleuthkit.autopsy.filetypeid - - - - org.netbeans.api.progress - - - - 1 - 1.32.1 - - - - org.netbeans.modules.options.api - - - - 1 - 1.31.2 - - - - org.openide.awt - - - - 7.55.1 - - - - org.openide.dialogs - - - - 7.28.1 - - - - org.openide.nodes - - - - 7.33.2 - - - - org.openide.util - - - - 8.29.3 - - - - org.openide.util.lookup - - - - 8.19.1 - - - - org.openide.windows - - - - 6.60.1 - - - - org.sleuthkit.autopsy.core - - - - 9 - 7.0 - - - - org.sleuthkit.autopsy.corelibs - - - - 3 - 1.1 - - - - - org.sleuthkit.autopsy.filetypeid - - - ext/tika-core-1.2.jar - release/modules/ext/tika-core-1.2.jar - - - - diff --git a/FileTypeId/nbproject/suite.properties b/FileTypeId/nbproject/suite.properties deleted file mode 100644 index 29d7cc9bd6..0000000000 --- a/FileTypeId/nbproject/suite.properties +++ /dev/null @@ -1 +0,0 @@ -suite.dir=${basedir}/.. diff --git a/FileTypeId/src-alt/JMimeMagicFileTypeDetector.java b/FileTypeId/src-alt/JMimeMagicFileTypeDetector.java deleted file mode 100644 index 246fadc81e..0000000000 --- a/FileTypeId/src-alt/JMimeMagicFileTypeDetector.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Autopsy Forensic Browser - * - * Copyright 2011 - 2013 Basis Technology Corp. - * Contact: carrier sleuthkit org - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.sleuthkit.autopsy.filetypeid; - -import org.sleuthkit.datamodel.AbstractFile; -import net.sf.jmimemagic.Magic; -import net.sf.jmimemagic.MagicMatch; -import net.sf.jmimemagic.MagicMatchNotFoundException; -import org.openide.util.Exceptions; - -/** - * - */ -public class JMimeMagicFileTypeDetector implements FileTypeDetectionInterface { - - @Override - public FileIdInfo attemptMatch(AbstractFile abstractFile) { - try { - FileIdInfo ret = new FileIdInfo(); - final int maxBytesInitial = 3000; //how many bytes to read on first pass - byte buffer[] = new byte[maxBytesInitial]; - ///@todo decide to use max bytes or give the whole file - int len = abstractFile.read(buffer, 0, maxBytesInitial); - - try { - MagicMatch match = Magic.getMagicMatch(buffer); - if (match != null) { - String matchStr = match.getMimeType(); - if (matchStr.equals("???")) { - String desc = match.getDescription(); - if (!desc.isEmpty()) { - ret.type = desc; - } - } else { - ret.type = matchStr; - } - ret.extension = match.getExtension(); - } - } catch (MagicMatchNotFoundException ex) { - //do nothing - } - - return ret; - - } catch (Exception ex) { - Exceptions.printStackTrace(ex); - return new FileIdInfo(); - } - } - -} diff --git a/FileTypeId/src-alt/MimeUtilFileTypeDetector.java b/FileTypeId/src-alt/MimeUtilFileTypeDetector.java deleted file mode 100644 index 40d5a74b58..0000000000 --- a/FileTypeId/src-alt/MimeUtilFileTypeDetector.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Autopsy Forensic Browser - * - * Copyright 2011 - 2013 Basis Technology Corp. - * Contact: carrier sleuthkit org - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.sleuthkit.autopsy.filetypeid; - -import eu.medsea.mimeutil.MimeException; -import eu.medsea.mimeutil.MimeType; -import eu.medsea.mimeutil.detector.MagicMimeMimeDetector; -import java.util.Iterator; -import java.util.LinkedHashSet; -import org.openide.util.Exceptions; -import org.sleuthkit.datamodel.AbstractFile; - -/** - * - */ -public class MimeUtilFileTypeDetector implements FileTypeDetectionInterface { - private static MagicMimeMimeDetector mimeUtil = new MagicMimeMimeDetector(); - - - @Override - public FileIdInfo attemptMatch(AbstractFile abstractFile) { - try { - FileIdInfo ret = new FileIdInfo(); - final int maxBytesInitial = 3000; //how many bytes to read on first pass - byte buffer[] = new byte[maxBytesInitial]; - int len = abstractFile.read(buffer, 0, maxBytesInitial); - - try { - LinkedHashSet mimeSet = (LinkedHashSet)mimeUtil.getMimeTypesByteArray(buffer); - - Iterator it = mimeSet.iterator(); - while (it.hasNext()) { - MimeType mt = (MimeType)it.next(); - ret.type = mt.getMediaType() + "/" + mt.getSubType(); - break; //just take the first one for now - } - - } catch (MimeException ex) { - //do nothing - } - - return ret; - - } catch (Exception ex) { - Exceptions.printStackTrace(ex); - return new FileIdInfo(); - } - } - -} diff --git a/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbIngestModule.java b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbIngestModule.java index 45dad3ccca..e623b4f561 100644 --- a/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbIngestModule.java +++ b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbIngestModule.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.concurrent.atomic.AtomicLong; import java.util.logging.Level; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.casemodule.Case; @@ -42,12 +43,12 @@ import org.sleuthkit.datamodel.TskException; import org.sleuthkit.autopsy.hashdatabase.HashDbManager.HashDb; import org.sleuthkit.autopsy.ingest.IngestModuleAdapter; import org.sleuthkit.autopsy.ingest.FileIngestModule; +import org.sleuthkit.autopsy.ingest.IngestModuleReferenceCounter; import org.sleuthkit.datamodel.HashInfo; public class HashDbIngestModule extends IngestModuleAdapter implements FileIngestModule { private static final Logger logger = Logger.getLogger(HashDbIngestModule.class.getName()); private static final int MAX_COMMENT_SIZE = 500; - private static int messageId = 0; // RJCTODO: This is not thread safe private final IngestServices services = IngestServices.getInstance(); private final Hash hasher = new Hash(); private final SleuthkitCase skCase = Case.getCurrentCase().getSleuthkitCase(); @@ -55,9 +56,11 @@ public class HashDbIngestModule extends IngestModuleAdapter implements FileInges private final HashLookupModuleSettings settings; private List knownBadHashSets = new ArrayList<>(); private List knownHashSets = new ArrayList<>(); - private int knownBadCount = 0; - private long calctime = 0; - private long lookuptime = 0; + private long jobId; + private static AtomicLong totalKnownBadCount = new AtomicLong(0); + private static AtomicLong totalCalctime = new AtomicLong(0); + private static AtomicLong totalLookuptime = new AtomicLong(0); + private static final IngestModuleReferenceCounter refCounter = new IngestModuleReferenceCounter(); HashDbIngestModule(HashLookupModuleSettings settings) { this.settings = settings; @@ -65,24 +68,30 @@ public class HashDbIngestModule extends IngestModuleAdapter implements FileInges @Override public void startUp(org.sleuthkit.autopsy.ingest.IngestJobContext context) throws IngestModuleException { + jobId = context.getJobId(); getEnabledHashSets(hashDbManager.getKnownBadFileHashSets(), knownBadHashSets); - if (knownBadHashSets.isEmpty()) { - services.postMessage(IngestMessage.createWarningMessage(++messageId, - HashLookupModuleFactory.getModuleName(), - NbBundle.getMessage(this.getClass(), - "HashDbIngestModule.noKnownBadHashDbSetMsg"), - NbBundle.getMessage(this.getClass(), - "HashDbIngestModule.knownBadFileSearchWillNotExecuteWarn"))); - } - getEnabledHashSets(hashDbManager.getKnownFileHashSets(), knownHashSets); - if (knownHashSets.isEmpty()) { - services.postMessage(IngestMessage.createWarningMessage(++messageId, - HashLookupModuleFactory.getModuleName(), - NbBundle.getMessage(this.getClass(), - "HashDbIngestModule.noKnownHashDbSetMsg"), - NbBundle.getMessage(this.getClass(), - "HashDbIngestModule.knownFileSearchWillNotExecuteWarn"))); + + if (refCounter.incrementAndGet(jobId) == 1) { + // if first module for this job then post error msgs if needed + + if (knownBadHashSets.isEmpty()) { + services.postMessage(IngestMessage.createWarningMessage( + HashLookupModuleFactory.getModuleName(), + NbBundle.getMessage(this.getClass(), + "HashDbIngestModule.noKnownBadHashDbSetMsg"), + NbBundle.getMessage(this.getClass(), + "HashDbIngestModule.knownBadFileSearchWillNotExecuteWarn"))); + } + + if (knownHashSets.isEmpty()) { + services.postMessage(IngestMessage.createWarningMessage( + HashLookupModuleFactory.getModuleName(), + NbBundle.getMessage(this.getClass(), + "HashDbIngestModule.noKnownHashDbSetMsg"), + NbBundle.getMessage(this.getClass(), + "HashDbIngestModule.knownFileSearchWillNotExecuteWarn"))); + } } } @@ -121,10 +130,12 @@ public class HashDbIngestModule extends IngestModuleAdapter implements FileInges try { long calcstart = System.currentTimeMillis(); md5Hash = hasher.calculateMd5(file); - calctime += (System.currentTimeMillis() - calcstart); + long delta = (System.currentTimeMillis() - calcstart); + totalCalctime.addAndGet(delta); + } catch (IOException ex) { logger.log(Level.WARNING, "Error calculating hash of file " + name, ex); - services.postMessage(IngestMessage.createErrorMessage(++messageId, + services.postMessage(IngestMessage.createErrorMessage( HashLookupModuleFactory.getModuleName(), NbBundle.getMessage(this.getClass(), "HashDbIngestModule.fileReadErrorMsg", @@ -145,12 +156,13 @@ public class HashDbIngestModule extends IngestModuleAdapter implements FileInges HashInfo hashInfo = db.lookUp(file); if (null != hashInfo) { foundBad = true; - knownBadCount += 1; + totalKnownBadCount.incrementAndGet(); + try { skCase.setKnown(file, TskData.FileKnown.BAD); } catch (TskException ex) { logger.log(Level.WARNING, "Couldn't set known bad state for file " + name + " - see sleuthkit log for details", ex); - services.postMessage(IngestMessage.createErrorMessage(++messageId, + services.postMessage(IngestMessage.createErrorMessage( HashLookupModuleFactory.getModuleName(), NbBundle.getMessage(this.getClass(), "HashDbIngestModule.hashLookupErrorMsg", @@ -178,10 +190,12 @@ public class HashDbIngestModule extends IngestModuleAdapter implements FileInges postHashSetHitToBlackboard(file, md5Hash, hashSetName, comment, db.getSendIngestMessages()); } - lookuptime += (System.currentTimeMillis() - lookupstart); + long delta = (System.currentTimeMillis() - lookupstart); + totalLookuptime.addAndGet(delta); + } catch (TskException ex) { logger.log(Level.WARNING, "Couldn't lookup known bad hash for file " + name + " - see sleuthkit log for details", ex); - services.postMessage(IngestMessage.createErrorMessage(++messageId, + services.postMessage(IngestMessage.createErrorMessage( HashLookupModuleFactory.getModuleName(), NbBundle.getMessage(this.getClass(), "HashDbIngestModule.hashLookupErrorMsg", @@ -209,10 +223,12 @@ public class HashDbIngestModule extends IngestModuleAdapter implements FileInges ret = ProcessResult.ERROR; } } - lookuptime += (System.currentTimeMillis() - lookupstart); + long delta = (System.currentTimeMillis() - lookupstart); + totalLookuptime.addAndGet(delta); + } catch (TskException ex) { logger.log(Level.WARNING, "Couldn't lookup known hash for file " + name + " - see sleuthkit log for details", ex); - services.postMessage(IngestMessage.createErrorMessage(++messageId, + services.postMessage(IngestMessage.createErrorMessage( HashLookupModuleFactory.getModuleName(), NbBundle.getMessage(this.getClass(), "HashDbIngestModule.hashLookupErrorMsg", @@ -272,7 +288,7 @@ public class HashDbIngestModule extends IngestModuleAdapter implements FileInges detailsSb.append(""); - services.postMessage(IngestMessage.createDataMessage(++messageId, HashLookupModuleFactory.getModuleName(), + services.postMessage(IngestMessage.createDataMessage( HashLookupModuleFactory.getModuleName(), NbBundle.getMessage(this.getClass(), "HashDbIngestModule.postToBB.knownBadMsg", abstractFile.getName()), @@ -286,41 +302,43 @@ public class HashDbIngestModule extends IngestModuleAdapter implements FileInges } } - + @Override public void shutDown(boolean ingestJobCancelled) { - if ((!knownBadHashSets.isEmpty()) || (!knownHashSets.isEmpty())) { - StringBuilder detailsSb = new StringBuilder(); - //details - detailsSb.append(""); + if (refCounter.decrementAndGet(jobId) == 0) { + if ((!knownBadHashSets.isEmpty()) || (!knownHashSets.isEmpty())) { + StringBuilder detailsSb = new StringBuilder(); + //details + detailsSb.append("
"); - detailsSb.append(""); - detailsSb.append(""); + detailsSb.append(""); + detailsSb.append(""); - detailsSb.append("\n"); - detailsSb.append("\n"); - detailsSb.append("
") - .append(NbBundle.getMessage(this.getClass(), "HashDbIngestModule.complete.knownBadsFound")) - .append("").append(knownBadCount).append("
") + .append(NbBundle.getMessage(this.getClass(), "HashDbIngestModule.complete.knownBadsFound")) + .append("").append(totalKnownBadCount.get()).append("
") - .append(NbBundle.getMessage(this.getClass(), "HashDbIngestModule.complete.totalCalcTime")) - .append("").append(calctime).append("
") - .append(NbBundle.getMessage(this.getClass(), "HashDbIngestModule.complete.totalLookupTime")) - .append("").append(lookuptime).append("
"); + detailsSb.append("") + .append(NbBundle.getMessage(this.getClass(), "HashDbIngestModule.complete.totalCalcTime")) + .append("").append(totalCalctime.get()).append("\n"); + detailsSb.append("") + .append(NbBundle.getMessage(this.getClass(), "HashDbIngestModule.complete.totalLookupTime")) + .append("").append(totalLookuptime.get()).append("\n"); + detailsSb.append(""); - detailsSb.append("

") - .append(NbBundle.getMessage(this.getClass(), "HashDbIngestModule.complete.databasesUsed")) - .append("

\n
    "); - for (HashDb db : knownBadHashSets) { - detailsSb.append("
  • ").append(db.getHashSetName()).append("
  • \n"); + detailsSb.append("

    ") + .append(NbBundle.getMessage(this.getClass(), "HashDbIngestModule.complete.databasesUsed")) + .append("

    \n
      "); + for (HashDb db : knownBadHashSets) { + detailsSb.append("
    • ").append(db.getHashSetName()).append("
    • \n"); + } + + detailsSb.append("
    "); + services.postMessage(IngestMessage.createMessage( + IngestMessage.MessageType.INFO, + HashLookupModuleFactory.getModuleName(), + NbBundle.getMessage(this.getClass(), + "HashDbIngestModule.complete.hashLookupResults"), + detailsSb.toString())); } - - detailsSb.append("
"); - services.postMessage(IngestMessage.createMessage(++messageId, - IngestMessage.MessageType.INFO, - HashLookupModuleFactory.getModuleName(), - NbBundle.getMessage(this.getClass(), - "HashDbIngestModule.complete.hashLookupResults"), - detailsSb.toString())); } } } diff --git a/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashLookupSettingsPanel.java b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashLookupSettingsPanel.java index 02b307fc8f..4e1e9ceb70 100644 --- a/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashLookupSettingsPanel.java +++ b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashLookupSettingsPanel.java @@ -20,6 +20,7 @@ package org.sleuthkit.autopsy.hashdatabase; import java.awt.Color; import java.awt.Component; +import java.awt.EventQueue; import java.awt.Frame; import java.awt.event.KeyEvent; import java.beans.PropertyChangeEvent; @@ -71,8 +72,13 @@ public final class HashLookupSettingsPanel extends IngestModuleGlobalSetttingsPa IngestManager.addPropertyChangeListener(new PropertyChangeListener() { @Override public void propertyChange(PropertyChangeEvent evt) { - if (isFileIngestStatusChangeEvent(evt)) { - updateComponents(); + if (isIngestJobEvent(evt)) { + EventQueue.invokeLater(new Runnable() { + @Override + public void run() { + updateComponents(); + } + }); } } }); @@ -224,8 +230,10 @@ public final class HashLookupSettingsPanel extends IngestModuleGlobalSetttingsPa return shortenedPath; } - private boolean isFileIngestStatusChangeEvent(PropertyChangeEvent evt) { - return evt.getPropertyName().equals(IngestManager.IngestEvent.STARTED.toString()) || evt.getPropertyName().equals(IngestManager.IngestEvent.COMPLETED.toString()) || evt.getPropertyName().equals(IngestManager.IngestEvent.STOPPED.toString()); + private boolean isIngestJobEvent(PropertyChangeEvent evt) { + return evt.getPropertyName().equals(IngestManager.IngestEvent.INGEST_JOB_STARTED.toString()) + || evt.getPropertyName().equals(IngestManager.IngestEvent.INGEST_JOB_COMPLETED.toString()) + || evt.getPropertyName().equals(IngestManager.IngestEvent.INGEST_JOB_CANCELLED.toString()); } @Override diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AbstractFileHtmlExtract.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AbstractFileHtmlExtract.java index e013ec2fac..7af85f7e3f 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AbstractFileHtmlExtract.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AbstractFileHtmlExtract.java @@ -177,10 +177,6 @@ import org.sleuthkit.datamodel.ReadContentInputStream; + sourceFile.getName() + "' (id: " + sourceFile.getId() + ").", ingEx); throw ingEx; //need to rethrow/return to signal error and move on } - - //check if need invoke commit/search between chunks - //not to delay commit if timer has gone off - module.checkRunCommitSearch(); } } catch (IOException ex) { logger.log(Level.WARNING, "Unable to read content stream from " + sourceFile.getId() + ": " + sourceFile.getName(), ex); diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AbstractFileStringExtract.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AbstractFileStringExtract.java index 4d82827c6a..6f96ff2495 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AbstractFileStringExtract.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AbstractFileStringExtract.java @@ -147,10 +147,6 @@ class AbstractFileStringExtract implements AbstractFileExtract { throw ingEx; //need to rethrow/return to signal error and move on } - //check if need invoke commit/search between chunks - //not to delay commit if timer has gone off - module.checkRunCommitSearch(); - //debug.close(); } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AbstractFileTikaTextExtract.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AbstractFileTikaTextExtract.java index 4f52805753..e19be18a5c 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AbstractFileTikaTextExtract.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AbstractFileTikaTextExtract.java @@ -223,10 +223,6 @@ class AbstractFileTikaTextExtract implements AbstractFileExtract { + sourceFile.getName() + "' (id: " + sourceFile.getId() + ").", ingEx); throw ingEx; //need to rethrow/return to signal error and move on } - - //check if need invoke commit/search between chunks - //not to delay commit if timer has gone off - module.checkRunCommitSearch(); } } catch (IOException ex) { final String msg = "Exception: Unable to read Tika content stream from " + sourceFile.getId() + ": " + sourceFile.getName(); diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties index 040e9e0c03..a608ce590f 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties @@ -263,3 +263,10 @@ KeywordSearchGlobalSearchSettingsPanel.timeRadioButton4.text_1=1 minute (faster KeywordSearchGlobalSearchSettingsPanel.chunksLabel.text=Chunks in keyword index: KeywordSearchGlobalSearchSettingsPanel.timeRadioButton3.toolTipText=5 minutes (overall ingest time will be longer) KeywordSearchGlobalSearchSettingsPanel.timeRadioButton3.text=5 minutes (default) +KeywordSearchIngestModule.regExpHitLbl=Reg Ex hit: +KeywordSearchIngestModule.kwHitLbl=Keyword hit: +KeywordSearchIngestModule.kwHitThLbl=Keyword +KeywordSearchIngestModule.previewThLbl=Preview +KeywordSearchIngestModule.fileThLbl=File +KeywordSearchIngestModule.listThLbl=List +KeywordSearchIngestModule.regExThLbl=Reg Ex diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Ingester.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Ingester.java index 12d9999b01..bdb304dabf 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Ingester.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Ingester.java @@ -53,7 +53,6 @@ import org.sleuthkit.datamodel.ContentVisitor; import org.sleuthkit.datamodel.DerivedFile; import org.sleuthkit.datamodel.Directory; import org.sleuthkit.datamodel.File; -import org.sleuthkit.datamodel.FsContent; import org.sleuthkit.datamodel.LayoutFile; import org.sleuthkit.datamodel.LocalFile; import org.sleuthkit.datamodel.ReadContentInputStream; @@ -66,7 +65,7 @@ import org.sleuthkit.datamodel.TskCoreException; class Ingester { private static final Logger logger = Logger.getLogger(Ingester.class.getName()); - private boolean uncommitedIngests = false; + private volatile boolean uncommitedIngests = false; private final ExecutorService upRequestExecutor = Executors.newSingleThreadExecutor(); private final Server solrServer = KeywordSearch.getServer(); private final GetContentFieldsV getContentFieldsV = new GetContentFieldsV(); @@ -74,8 +73,7 @@ class Ingester { //for ingesting chunk as SolrInputDocument (non-content-streaming, by-pass tika) //TODO use a streaming way to add content to /update handler - private final static int MAX_DOC_CHUNK_SIZE = 1024*1024; - private final byte[] docChunkContentBuf = new byte[MAX_DOC_CHUNK_SIZE]; + private static final int MAX_DOC_CHUNK_SIZE = 1024*1024; private static final String docContentEncoding = "UTF-8"; @@ -286,6 +284,7 @@ class Ingester { throw new IngesterException(msg); } + final byte[] docChunkContentBuf = new byte[MAX_DOC_CHUNK_SIZE]; SolrInputDocument updateDoc = new SolrInputDocument(); for (String key : fields.keySet()) { diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchEditListPanel.form b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchEditListPanel.form index 1f31418be7..83675f4797 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchEditListPanel.form +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchEditListPanel.form @@ -39,14 +39,6 @@ - - - - - - - - @@ -67,7 +59,10 @@ - + + + + @@ -114,7 +109,7 @@ - + @@ -128,8 +123,9 @@ - - + + + @@ -146,15 +142,15 @@ - + - + - + diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchEditListPanel.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchEditListPanel.java index 0014a785e0..27ba1933e5 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchEditListPanel.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchEditListPanel.java @@ -18,6 +18,7 @@ */ package org.sleuthkit.autopsy.keywordsearch; +import java.awt.EventQueue; import java.awt.event.ActionEvent; import java.awt.event.ActionListener; import java.beans.PropertyChangeEvent; @@ -52,7 +53,6 @@ class KeywordSearchEditListPanel extends javax.swing.JPanel implements ListSelec private static Logger logger = Logger.getLogger(KeywordSearchEditListPanel.class.getName()); private KeywordTableModel tableModel; private KeywordList currentKeywordList; - private boolean ingestRunning; /** * Creates new form KeywordSearchEditListPanel @@ -101,7 +101,7 @@ class KeywordSearchEditListPanel extends javax.swing.JPanel implements ListSelec } }); - initButtons(); + setButtonStates(); addWordField.setComponentPopupMenu(rightClickMenu); ActionListener actList = new ActionListener() { @@ -124,49 +124,28 @@ class KeywordSearchEditListPanel extends javax.swing.JPanel implements ListSelec pasteMenuItem.addActionListener(actList); selectAllMenuItem.addActionListener(actList); - if (IngestManager.getInstance().isIngestRunning()) { - initIngest(0); - } else { - initIngest(1); - } + setButtonStates(); IngestManager.addPropertyChangeListener(new PropertyChangeListener() { @Override public void propertyChange(PropertyChangeEvent evt) { String changed = evt.getPropertyName(); - Object oldValue = evt.getOldValue(); - if (changed.equals(IngestEvent.COMPLETED.toString()) - && ((String) oldValue).equals(KeywordSearchModuleFactory.getModuleName())) { - initIngest(1); - } else if (changed.equals(IngestEvent.STARTED.toString()) - && ((String) oldValue).equals(KeywordSearchModuleFactory.getModuleName())) { - initIngest(0); - } else if (changed.equals(IngestEvent.STOPPED.toString()) - && ((String) oldValue).equals(KeywordSearchModuleFactory.getModuleName())) { - initIngest(1); + if (changed.equals(IngestEvent.INGEST_JOB_STARTED.toString()) + || changed.equals(IngestEvent.INGEST_JOB_COMPLETED.toString()) + || changed.equals(IngestEvent.INGEST_JOB_CANCELLED.toString())) { + EventQueue.invokeLater(new Runnable() { + @Override + public void run() { + setButtonStates(); + } + }); } } }); } - /** - * Initialize this panel depending on whether ingest is running - * - * @param running case 0: ingest running case 1: ingest not running - */ - private void initIngest(int running) { - switch (running) { - case 0: - ingestRunning = true; - break; - case 1: - ingestRunning = false; - break; - } - initButtons(); - } - - void initButtons() { + void setButtonStates() { + boolean ingestRunning = IngestManager.getInstance().isIngestRunning(); boolean listSet = currentKeywordList != null; boolean isLocked = !listSet ? true : currentKeywordList.isLocked(); boolean noKeywords = !listSet ? true : currentKeywordList.getKeywords().isEmpty(); @@ -237,9 +216,6 @@ class KeywordSearchEditListPanel extends javax.swing.JPanel implements ListSelec selectAllMenuItem.setText(org.openide.util.NbBundle.getMessage(KeywordSearchEditListPanel.class, "KeywordSearchEditListPanel.selectAllMenuItem.text")); // NOI18N rightClickMenu.add(selectAllMenuItem); - setMinimumSize(new java.awt.Dimension(340, 300)); - setPreferredSize(new java.awt.Dimension(340, 420)); - jScrollPane1.setPreferredSize(new java.awt.Dimension(340, 300)); keywordTable.setModel(tableModel); @@ -363,7 +339,7 @@ class KeywordSearchEditListPanel extends javax.swing.JPanel implements ListSelec .addComponent(saveListButton) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) .addComponent(deleteListButton))))) - .addGap(0, 44, Short.MAX_VALUE))) + .addGap(0, 0, Short.MAX_VALUE))) .addContainerGap()))) ); listEditorPanelLayout.setVerticalGroup( @@ -372,7 +348,8 @@ class KeywordSearchEditListPanel extends javax.swing.JPanel implements ListSelec .addContainerGap() .addComponent(keywordsLabel) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) - .addComponent(jScrollPane1, javax.swing.GroupLayout.PREFERRED_SIZE, 188, javax.swing.GroupLayout.PREFERRED_SIZE) + .addComponent(jScrollPane1, javax.swing.GroupLayout.PREFERRED_SIZE, 178, javax.swing.GroupLayout.PREFERRED_SIZE) + .addGap(10, 10, 10) .addGroup(listEditorPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addGroup(listEditorPanelLayout.createSequentialGroup() .addGroup(listEditorPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.TRAILING) @@ -385,14 +362,14 @@ class KeywordSearchEditListPanel extends javax.swing.JPanel implements ListSelec .addGroup(listEditorPanelLayout.createSequentialGroup() .addGap(123, 123, 123) .addComponent(listOptionsSeparator, javax.swing.GroupLayout.PREFERRED_SIZE, 6, javax.swing.GroupLayout.PREFERRED_SIZE))) - .addGap(18, 18, 18) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) .addComponent(ingestMessagesCheckbox) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED, 13, Short.MAX_VALUE) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) .addGroup(listEditorPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) .addComponent(exportButton) .addComponent(saveListButton) .addComponent(deleteListButton)) - .addGap(42, 42, 42)) + .addContainerGap()) ); javax.swing.GroupLayout layout = new javax.swing.GroupLayout(this); @@ -403,7 +380,9 @@ class KeywordSearchEditListPanel extends javax.swing.JPanel implements ListSelec ); layout.setVerticalGroup( layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addComponent(listEditorPanel, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) + .addGroup(layout.createSequentialGroup() + .addComponent(listEditorPanel, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE) + .addContainerGap(javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)) ); }// //GEN-END:initComponents @@ -441,7 +420,7 @@ class KeywordSearchEditListPanel extends javax.swing.JPanel implements ListSelec chRegex.setSelected(false); addWordField.setText(""); - initButtons(); + setButtonStates(); }//GEN-LAST:event_addWordButtonActionPerformed private void deleteWordButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_deleteWordButtonActionPerformed @@ -449,7 +428,7 @@ class KeywordSearchEditListPanel extends javax.swing.JPanel implements ListSelec tableModel.deleteSelected(keywordTable.getSelectedRows()); KeywordSearchListsXML.getCurrent().addList(currentKeywordList); - initButtons(); + setButtonStates(); } }//GEN-LAST:event_deleteWordButtonActionPerformed @@ -549,11 +528,11 @@ class KeywordSearchEditListPanel extends javax.swing.JPanel implements ListSelec currentKeywordList = loader.getListsL(false).get(index); tableModel.resync(); - initButtons(); + setButtonStates(); } else { currentKeywordList = null; tableModel.resync(); - initButtons(); + setButtonStates(); } } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalListSettingsPanel.form b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalListSettingsPanel.form index bad2ca29c9..e0660ba6b1 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalListSettingsPanel.form +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalListSettingsPanel.form @@ -16,7 +16,7 @@ - + @@ -57,6 +57,11 @@ + + + + + @@ -66,7 +71,7 @@ - + diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalListSettingsPanel.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalListSettingsPanel.java index 36b22ef853..ac8e74212d 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalListSettingsPanel.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalListSettingsPanel.java @@ -44,7 +44,7 @@ final class KeywordSearchGlobalListSettingsPanel extends javax.swing.JPanel impl if (KeywordSearchUtil.displayConfirmDialog(NbBundle.getMessage(this.getClass(), "KeywordSearchConfigurationPanel1.customizeComponents.title"), NbBundle.getMessage(this.getClass(), "KeywordSearchConfigurationPanel1.customizeComponents.body"), KeywordSearchUtil.DIALOG_MESSAGE_TYPE.WARN)) { String toDelete = editListPanel.getCurrentKeywordList().getName(); editListPanel.setCurrentKeywordList(null); - editListPanel.initButtons(); + editListPanel.setButtonStates(); // RJCTODO: Move this into a deleteList method in the manager KeywordSearchListsXML deleter = KeywordSearchListsXML.getCurrent(); deleter.deleteList(toDelete); @@ -152,11 +152,13 @@ final class KeywordSearchGlobalListSettingsPanel extends javax.swing.JPanel impl mainSplitPane.setLeftComponent(leftPanel); + rightPanel.setPreferredSize(new java.awt.Dimension(360, 327)); + javax.swing.GroupLayout rightPanelLayout = new javax.swing.GroupLayout(rightPanel); rightPanel.setLayout(rightPanelLayout); rightPanelLayout.setHorizontalGroup( rightPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addGap(0, 318, Short.MAX_VALUE) + .addGap(0, 385, Short.MAX_VALUE) ); rightPanelLayout.setVerticalGroup( rightPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) @@ -169,7 +171,7 @@ final class KeywordSearchGlobalListSettingsPanel extends javax.swing.JPanel impl this.setLayout(layout); layout.setHorizontalGroup( layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addComponent(mainSplitPane) + .addComponent(mainSplitPane, javax.swing.GroupLayout.DEFAULT_SIZE, 665, Short.MAX_VALUE) ); layout.setVerticalGroup( layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalSettingsPanel.form b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalSettingsPanel.form index 6a22cf2579..a660de43de 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalSettingsPanel.form +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalSettingsPanel.form @@ -16,12 +16,12 @@ - + - + diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalSettingsPanel.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalSettingsPanel.java index 44e1dc22a1..d400abe80c 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalSettingsPanel.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalSettingsPanel.java @@ -63,11 +63,11 @@ final class KeywordSearchGlobalSettingsPanel extends IngestModuleGlobalSetttings this.setLayout(layout); layout.setHorizontalGroup( layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addComponent(tabbedPane, javax.swing.GroupLayout.DEFAULT_SIZE, 400, Short.MAX_VALUE) + .addComponent(tabbedPane, javax.swing.GroupLayout.DEFAULT_SIZE, 670, Short.MAX_VALUE) ); layout.setVerticalGroup( layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addComponent(tabbedPane, javax.swing.GroupLayout.DEFAULT_SIZE, 300, Short.MAX_VALUE) + .addComponent(tabbedPane, javax.swing.GroupLayout.DEFAULT_SIZE, 310, Short.MAX_VALUE) ); }// //GEN-END:initComponents diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java old mode 100755 new mode 100644 index 867e59eb16..dbb9c198e5 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java @@ -18,36 +18,19 @@ */ package org.sleuthkit.autopsy.keywordsearch; -import java.awt.event.ActionEvent; -import java.awt.event.ActionListener; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; -import java.util.Collection; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Set; -import java.util.concurrent.CancellationException; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Level; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.coreutils.Logger; -import javax.swing.SwingUtilities; -import javax.swing.SwingWorker; -import javax.swing.Timer; import org.apache.tika.Tika; -import org.netbeans.api.progress.aggregate.AggregateProgressFactory; -import org.netbeans.api.progress.aggregate.AggregateProgressHandle; -import org.netbeans.api.progress.aggregate.ProgressContributor; -import org.openide.util.Cancellable; import org.sleuthkit.autopsy.casemodule.Case; -import org.sleuthkit.autopsy.coreutils.EscapeUtil; import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; -import org.sleuthkit.autopsy.coreutils.StopWatch; import org.sleuthkit.autopsy.coreutils.StringExtract.StringExtractUnicodeTable.SCRIPT; import org.sleuthkit.autopsy.ingest.FileIngestModule; import org.sleuthkit.autopsy.ingest.IngestServices; @@ -55,11 +38,8 @@ import org.sleuthkit.autopsy.ingest.IngestMessage; import org.sleuthkit.autopsy.ingest.IngestMessage.MessageType; import org.sleuthkit.autopsy.ingest.IngestModuleAdapter; import org.sleuthkit.autopsy.ingest.IngestJobContext; -import org.sleuthkit.autopsy.ingest.ModuleDataEvent; +import org.sleuthkit.autopsy.ingest.IngestModuleReferenceCounter; import org.sleuthkit.autopsy.keywordsearch.Ingester.IngesterException; -import org.sleuthkit.datamodel.BlackboardArtifact; -import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE; -import org.sleuthkit.datamodel.BlackboardAttribute; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.ReadContentInputStream; import org.sleuthkit.datamodel.SleuthkitCase; @@ -97,29 +77,23 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme private static final Logger logger = Logger.getLogger(KeywordSearchIngestModule.class.getName()); private IngestServices services = IngestServices.getInstance(); private Ingester ingester = null; - private volatile boolean commitIndex = false; //whether to commit index next time - private volatile boolean runSearcher = false; //whether to run searcher next time - private Timer commitTimer; - private Timer searchTimer; private Indexer indexer; - private Searcher currentSearcher; - private Searcher finalSearcher; - private volatile boolean searcherDone = true; //mark as done, until it's inited - private Map> currentResults; //only search images from current ingest, not images previously ingested/indexed //accessed read-only by searcher thread - private Set curDataSourceIds; - private static final ReentrantReadWriteLock rwLock = new ReentrantReadWriteLock(true); //use fairness policy - private static final Lock searcherLock = rwLock.writeLock(); - private volatile int messageID = 0; // RJCTODO: Despite volatile, this is not thread safe, uses increment (not atomic) - private boolean processedFiles; + + private boolean startedSearching = false; private SleuthkitCase caseHandle = null; - private static List textExtractors; - private static AbstractFileStringExtract stringExtractor; + private List textExtractors; + private AbstractFileStringExtract stringExtractor; private final KeywordSearchJobSettings settings; private boolean initialized = false; private Tika tikaFormatDetector; - + private long jobId; + private long dataSourceId; + private static AtomicInteger instanceCount = new AtomicInteger(0); //just used for logging + private int instanceNum = 0; + private static final IngestModuleReferenceCounter refCounter = new IngestModuleReferenceCounter(); + private enum IngestStatus { TEXT_INGESTED, /// Text was extracted by knowing file type and text_ingested @@ -129,10 +103,17 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme SKIPPED_ERROR_TEXTEXTRACT, ///< File was skipped because of text extraction issues SKIPPED_ERROR_IO ///< File was skipped because of IO issues reading it }; - private Map ingestStatus; + private static final Map ingestStatus = new HashMap<>(); //guarded by itself + static void putIngestStatus(long id, IngestStatus status) { + synchronized(ingestStatus) { + ingestStatus.put(id, status); + } + } + KeywordSearchIngestModule(KeywordSearchJobSettings settings) { this.settings = settings; + instanceNum = instanceCount.getAndIncrement(); } /** @@ -142,42 +123,58 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme */ @Override public void startUp(IngestJobContext context) throws IngestModuleException { - logger.log(Level.INFO, "init()"); - initialized = false; - + logger.log(Level.INFO, "Initializing instance {0}", instanceNum); + initialized = false; + jobId = context.getJobId(); caseHandle = Case.getCurrentCase().getSleuthkitCase(); - tikaFormatDetector = new Tika(); - ingester = Server.getIngester(); - final Server server = KeywordSearch.getServer(); - try { - if (!server.isRunning()) { + // increment the module reference count + // if first instance of this module for this job then check the server and existence of keywords + if (refCounter.incrementAndGet(jobId) == 1) { + final Server server = KeywordSearch.getServer(); + try { + if (!server.isRunning()) { + String msg = NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.badInitMsg"); + logger.log(Level.SEVERE, msg); + String details = NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.tryStopSolrMsg", msg); + services.postMessage(IngestMessage.createErrorMessage(KeywordSearchModuleFactory.getModuleName(), msg, details)); + throw new IngestModuleException(msg); + } + } catch (KeywordSearchModuleException ex) { + logger.log(Level.WARNING, "Error checking if Solr server is running while initializing ingest", ex); + //this means Solr is not properly initialized String msg = NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.badInitMsg"); - logger.log(Level.SEVERE, msg); String details = NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.tryStopSolrMsg", msg); - services.postMessage(IngestMessage.createErrorMessage(++messageID, KeywordSearchModuleFactory.getModuleName(), msg, details)); + services.postMessage(IngestMessage.createErrorMessage(KeywordSearchModuleFactory.getModuleName(), msg, details)); throw new IngestModuleException(msg); } - } catch (KeywordSearchModuleException ex) { - logger.log(Level.WARNING, "Error checking if Solr server is running while initializing ingest", ex); - //this means Solr is not properly initialized - String msg = NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.badInitMsg"); - String details = NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.tryStopSolrMsg", msg); - services.postMessage(IngestMessage.createErrorMessage(++messageID, KeywordSearchModuleFactory.getModuleName(), msg, details)); - throw new IngestModuleException(msg); - } - try { - // make an actual query to verify that server is responding - // we had cases where getStatus was OK, but the connection resulted in a 404 - server.queryNumIndexedDocuments(); - } catch (KeywordSearchModuleException | NoOpenCoreException ex) { - throw new IngestModuleException( - NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.exception.errConnToSolr.msg", - ex.getMessage())); - } + try { + // make an actual query to verify that server is responding + // we had cases where getStatus was OK, but the connection resulted in a 404 + server.queryNumIndexedDocuments(); + } catch (KeywordSearchModuleException | NoOpenCoreException ex) { + throw new IngestModuleException( + NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.exception.errConnToSolr.msg", + ex.getMessage())); + } + // check if this job has any searchable keywords + List keywordLists = KeywordSearchListsXML.getCurrent().getListsL(); + boolean hasKeywordsForSearch = false; + for (KeywordList keywordList : keywordLists) { + if (settings.isKeywordListEnabled(keywordList.getName()) && !keywordList.getKeywords().isEmpty()) { + hasKeywordsForSearch = true; + break; + } + } + if (!hasKeywordsForSearch) { + services.postMessage(IngestMessage.createWarningMessage(KeywordSearchModuleFactory.getModuleName(), NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.noKwInLstMsg"), + NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.onlyIdxKwSkipMsg"))); + } + } + //initialize extractors stringExtractor = new AbstractFileStringExtract(this); stringExtractor.setScripts(KeywordSearchSettings.getStringExtractScripts()); @@ -194,57 +191,22 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme //order matters, more specific extractors first textExtractors.add(new AbstractFileHtmlExtract(this)); textExtractors.add(new AbstractFileTikaTextExtract(this)); - - ingestStatus = new HashMap<>(); - - List keywordLists = KeywordSearchListsXML.getCurrent().getListsL(); - boolean hasKeywordsForSearch = false; - for (KeywordList keywordList : keywordLists) { - if (settings.isKeywordListEnabled(keywordList.getName()) && !keywordList.getKeywords().isEmpty()) { - hasKeywordsForSearch = true; - break; - } - } - if (!hasKeywordsForSearch) { - services.postMessage(IngestMessage.createWarningMessage(++messageID, KeywordSearchModuleFactory.getModuleName(), NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.noKwInLstMsg"), - NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.onlyIdxKwSkipMsg"))); - } - - processedFiles = false; - searcherDone = true; //make sure to start the initial currentSearcher - //keeps track of all results per run not to repeat reporting the same hits - currentResults = new HashMap<>(); - - curDataSourceIds = new HashSet<>(); - + indexer = new Indexer(); - - final int updateIntervalMs = KeywordSearchSettings.getUpdateFrequency().getTime() * 60 * 1000; - logger.log(Level.INFO, "Using commit interval (ms): {0}", updateIntervalMs); - logger.log(Level.INFO, "Using searcher interval (ms): {0}", updateIntervalMs); - - commitTimer = new Timer(updateIntervalMs, new CommitTimerAction()); - searchTimer = new Timer(updateIntervalMs, new SearchTimerAction()); - initialized = true; - - commitTimer.start(); - searchTimer.start(); } @Override public ProcessResult process(AbstractFile abstractFile) { - if (initialized == false) //error initializing indexing/Solr { logger.log(Level.WARNING, "Skipping processing, module not initialized, file: {0}", abstractFile.getName()); - ingestStatus.put(abstractFile.getId(), IngestStatus.SKIPPED_ERROR_INDEXING); + putIngestStatus(abstractFile.getId(), IngestStatus.SKIPPED_ERROR_INDEXING); return ProcessResult.OK; } try { //add data source id of the file to the set, keeping track of images being ingested - final long fileSourceId = caseHandle.getFileDataSource(abstractFile); - curDataSourceIds.add(fileSourceId); + dataSourceId = caseHandle.getFileDataSource(abstractFile); } catch (TskCoreException ex) { logger.log(Level.SEVERE, "Error getting image id of file processed by keyword search: " + abstractFile.getName(), ex); @@ -261,14 +223,16 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme return ProcessResult.OK; } - processedFiles = true; - - //check if it's time to commit after previous processing - checkRunCommitSearch(); - //index the file and content (if the content is supported) indexer.indexFile(abstractFile, true); + // Start searching if it hasn't started already + if (!startedSearching) { + List keywordListNames = settings.getNamesOfEnabledKeyWordLists(); + SearchRunner.getInstance().startJob(jobId, dataSourceId, keywordListNames); + startedSearching = true; + } + return ProcessResult.OK; } @@ -278,39 +242,26 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme */ @Override public void shutDown(boolean ingestJobCancelled) { + logger.log(Level.INFO, "Instance {0}", instanceNum); + if (initialized == false) { return; } if (ingestJobCancelled) { + logger.log(Level.INFO, "Ingest job cancelled"); stop(); return; } - commitTimer.stop(); - - //NOTE, we let the 1 before last searcher complete fully, and enqueue the last one - - //cancel searcher timer, ensure unwanted searcher does not start - //before we start the final one - if (searchTimer.isRunning()) { - searchTimer.stop(); + // Remove from the search list and trigger final commit and final search + SearchRunner.getInstance().endJob(jobId); + + // We only need to post the summary msg from the last module per job + if (refCounter.decrementAndGet(jobId) == 0) { + postIndexSummary(); } - runSearcher = false; - - logger.log(Level.INFO, "Running final index commit and search"); - //final commit - commit(); - - postIndexSummary(); - - //run one last search as there are probably some new files committed - List keywordLists = settings.getNamesOfEnabledKeyWordLists(); - if (!keywordLists.isEmpty() && processedFiles == true) { - finalSearcher = new Searcher(keywordLists, true); //final searcher run - finalSearcher.execute(); - } - + //log number of files / chunks in index //signal a potential change in number of text_ingested files try { @@ -321,8 +272,6 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme } catch (NoOpenCoreException | KeywordSearchModuleException ex) { logger.log(Level.WARNING, "Error executing Solr query to check number of indexed files/chunks: ", ex); } - - //cleanup done in final searcher } /** @@ -331,22 +280,8 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme private void stop() { logger.log(Level.INFO, "stop()"); - //stop timer - commitTimer.stop(); - //stop currentSearcher - if (currentSearcher != null) { - currentSearcher.cancel(true); - } - - //cancel searcher timer, ensure unwanted searcher does not start - if (searchTimer.isRunning()) { - searchTimer.stop(); - } - runSearcher = false; - - //commit uncommited files, don't search again - commit(); - + SearchRunner.getInstance().stopJob(jobId); + cleanup(); } @@ -354,16 +289,9 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme * Common cleanup code when module stops or final searcher completes */ private void cleanup() { - ingestStatus.clear(); - currentResults.clear(); - curDataSourceIds.clear(); - currentSearcher = null; - //finalSearcher = null; //do not collect, might be finalizing - - commitTimer.stop(); - searchTimer.stop(); - commitTimer = null; - //searchTimer = null; // do not collect, final searcher might still be running, in which case it throws an exception + synchronized(ingestStatus) { + ingestStatus.clear(); + } textExtractors.clear(); textExtractors = null; @@ -374,19 +302,6 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme initialized = false; } - /** - * Commits index and notifies listeners of index update - */ - private void commit() { - if (initialized) { - logger.log(Level.INFO, "Commiting index"); - ingester.commit(); - logger.log(Level.INFO, "Index comitted"); - //signal a potential change in number of text_ingested files - indexChangeNotify(); - } - } - /** * Posts inbox message with summary of text_ingested files */ @@ -397,31 +312,34 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme int error_text = 0; int error_index = 0; int error_io = 0; - for (IngestStatus s : ingestStatus.values()) { - switch (s) { - case TEXT_INGESTED: - ++text_ingested; - break; - case METADATA_INGESTED: - ++metadata_ingested; - break; - case STRINGS_INGESTED: - ++strings_ingested; - break; - case SKIPPED_ERROR_TEXTEXTRACT: - error_text++; - break; - case SKIPPED_ERROR_INDEXING: - error_index++; - break; - case SKIPPED_ERROR_IO: - error_io++; - break; - default: - ; + + synchronized(ingestStatus) { + for (IngestStatus s : ingestStatus.values()) { + switch (s) { + case TEXT_INGESTED: + ++text_ingested; + break; + case METADATA_INGESTED: + ++metadata_ingested; + break; + case STRINGS_INGESTED: + ++strings_ingested; + break; + case SKIPPED_ERROR_TEXTEXTRACT: + error_text++; + break; + case SKIPPED_ERROR_INDEXING: + error_index++; + break; + case SKIPPED_ERROR_IO: + error_io++; + break; + default: + ; + } } } - + StringBuilder msg = new StringBuilder(); msg.append(""); msg.append(""); @@ -432,7 +350,7 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme msg.append("
").append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.postIndexSummary.knowFileHeaderLbl")).append("").append(text_ingested).append("
").append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.postIndexSummary.fileGenStringsHead")).append("").append(strings_ingested).append("
"); String indexStats = msg.toString(); logger.log(Level.INFO, "Keyword Indexing Completed: {0}", indexStats); - services.postMessage(IngestMessage.createMessage(++messageID, MessageType.INFO, KeywordSearchModuleFactory.getModuleName(), NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.postIndexSummary.kwIdxResultsLbl"), indexStats)); + services.postMessage(IngestMessage.createMessage(MessageType.INFO, KeywordSearchModuleFactory.getModuleName(), NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.postIndexSummary.kwIdxResultsLbl"), indexStats)); if (error_index > 0) { MessageNotifyUtil.Notify.error(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.postIndexSummary.kwIdxErrsTitle"), NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.postIndexSummary.kwIdxErrMsgFiles", error_index)); @@ -442,73 +360,6 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme } } - /** - * Helper method to notify listeners on index update - */ - private void indexChangeNotify() { - //signal a potential change in number of text_ingested files - try { - final int numIndexedFiles = KeywordSearch.getServer().queryNumIndexedFiles(); - KeywordSearch.fireNumIndexedFilesChange(null, new Integer(numIndexedFiles)); - } catch (NoOpenCoreException | KeywordSearchModuleException ex) { - logger.log(Level.WARNING, "Error executing Solr query to check number of indexed files: ", ex); - } - } - - /** - * Check if time to commit, if so, run commit. Then run search if search - * timer is also set. - */ - void checkRunCommitSearch() { - if (commitIndex) { - logger.log(Level.INFO, "Commiting index"); - commit(); - commitIndex = false; - - //after commit, check if time to run searcher - //NOTE commit/searcher timings don't need to align - //in worst case, we will run search next time after commit timer goes off, or at the end of ingest - if (searcherDone && runSearcher) { - //start search if previous not running - List keywordLists = settings.getNamesOfEnabledKeyWordLists(); - if (!keywordLists.isEmpty()) { - currentSearcher = new Searcher(keywordLists); - currentSearcher.execute();//searcher will stop timer and restart timer when done - } - } - } - } - - /** - * CommitTimerAction to run by commitTimer Sets a flag to indicate we are - * ready for commit - */ - private class CommitTimerAction implements ActionListener { - - private final Logger logger = Logger.getLogger(CommitTimerAction.class.getName()); - - @Override - public void actionPerformed(ActionEvent e) { - commitIndex = true; - logger.log(Level.INFO, "CommitTimer awake"); - } - } - - /** - * SearchTimerAction to run by searchTimer Sets a flag to indicate we are - * ready to search - */ - private class SearchTimerAction implements ActionListener { - - private final Logger logger = Logger.getLogger(SearchTimerAction.class.getName()); - - @Override - public void actionPerformed(ActionEvent e) { - runSearcher = true; - logger.log(Level.INFO, "SearchTimer awake"); - } - } - /** * File indexer, processes and indexes known/allocated files, * unknown/unallocated files and directories accordingly @@ -560,16 +411,16 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme private boolean extractStringsAndIndex(AbstractFile aFile) { try { if (stringExtractor.index(aFile)) { - ingestStatus.put(aFile.getId(), IngestStatus.STRINGS_INGESTED); + putIngestStatus(aFile.getId(), IngestStatus.STRINGS_INGESTED); return true; } else { logger.log(Level.WARNING, "Failed to extract strings and ingest, file ''{0}'' (id: {1}).", new Object[]{aFile.getName(), aFile.getId()}); - ingestStatus.put(aFile.getId(), IngestStatus.SKIPPED_ERROR_TEXTEXTRACT); + putIngestStatus(aFile.getId(), IngestStatus.SKIPPED_ERROR_TEXTEXTRACT); return false; } } catch (IngesterException ex) { logger.log(Level.WARNING, "Failed to extract strings and ingest, file '" + aFile.getName() + "' (id: " + aFile.getId() + ").", ex); - ingestStatus.put(aFile.getId(), IngestStatus.SKIPPED_ERROR_INDEXING); + putIngestStatus(aFile.getId(), IngestStatus.SKIPPED_ERROR_INDEXING); return false; } } @@ -615,9 +466,9 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme if ((indexContent == false || aFile.isDir() || size == 0)) { try { ingester.ingest(aFile, false); //meta-data only - ingestStatus.put(aFile.getId(), IngestStatus.METADATA_INGESTED); + putIngestStatus(aFile.getId(), IngestStatus.METADATA_INGESTED); } catch (IngesterException ex) { - ingestStatus.put(aFile.getId(), IngestStatus.SKIPPED_ERROR_INDEXING); + putIngestStatus(aFile.getId(), IngestStatus.SKIPPED_ERROR_INDEXING); logger.log(Level.WARNING, "Unable to index meta-data for file: " + aFile.getId(), ex); } return; @@ -651,9 +502,9 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme if (AbstractFileExtract.ARCHIVE_MIME_TYPES.contains(detectedFormat)) { try { ingester.ingest(aFile, false); //meta-data only - ingestStatus.put(aFile.getId(), IngestStatus.METADATA_INGESTED); + putIngestStatus(aFile.getId(), IngestStatus.METADATA_INGESTED); } catch (IngesterException ex) { - ingestStatus.put(aFile.getId(), IngestStatus.SKIPPED_ERROR_INDEXING); + putIngestStatus(aFile.getId(), IngestStatus.SKIPPED_ERROR_INDEXING); logger.log(Level.WARNING, "Unable to index meta-data for file: " + aFile.getId(), ex); } return; @@ -666,20 +517,20 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme //logger.log(Level.INFO, "indexing: " + aFile.getName()); if (!extractTextAndIndex(aFile, detectedFormat)) { logger.log(Level.WARNING, "Failed to extract text and ingest, file ''{0}'' (id: {1}).", new Object[]{aFile.getName(), aFile.getId()}); - ingestStatus.put(aFile.getId(), IngestStatus.SKIPPED_ERROR_TEXTEXTRACT); + putIngestStatus(aFile.getId(), IngestStatus.SKIPPED_ERROR_TEXTEXTRACT); } else { - ingestStatus.put(aFile.getId(), IngestStatus.TEXT_INGESTED); + putIngestStatus(aFile.getId(), IngestStatus.TEXT_INGESTED); wasTextAdded = true; } } catch (IngesterException e) { logger.log(Level.INFO, "Could not extract text with Tika, " + aFile.getId() + ", " + aFile.getName(), e); - ingestStatus.put(aFile.getId(), IngestStatus.SKIPPED_ERROR_INDEXING); + putIngestStatus(aFile.getId(), IngestStatus.SKIPPED_ERROR_INDEXING); } catch (Exception e) { logger.log(Level.WARNING, "Error extracting text with Tika, " + aFile.getId() + ", " + aFile.getName(), e); - ingestStatus.put(aFile.getId(), IngestStatus.SKIPPED_ERROR_TEXTEXTRACT); + putIngestStatus(aFile.getId(), IngestStatus.SKIPPED_ERROR_TEXTEXTRACT); } } @@ -689,421 +540,4 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme } } } - - /** - * Searcher responsible for searching the current index and writing results - * to blackboard and the inbox. Also, posts results to listeners as Ingest - * data events. Searches entire index, and keeps track of only new results - * to report and save. Runs as a background thread. - */ - private final class Searcher extends SwingWorker { - - /** - * Searcher has private copies/snapshots of the lists and keywords - */ - private List keywords; //keywords to search - private List keywordLists; // lists currently being searched - private Map keywordToList; //keyword to list name mapping - private AggregateProgressHandle progressGroup; - private final Logger logger = Logger.getLogger(Searcher.class.getName()); - private boolean finalRun = false; - - Searcher(List keywordLists) { - this.keywordLists = new ArrayList<>(keywordLists); - this.keywords = new ArrayList<>(); - this.keywordToList = new HashMap<>(); - //keywords are populated as searcher runs - } - - Searcher(List keywordLists, boolean finalRun) { - this(keywordLists); - this.finalRun = finalRun; - } - - @Override - protected Object doInBackground() throws Exception { - if (finalRun) { - logger.log(Level.INFO, "Pending start of new (final) searcher"); - } else { - logger.log(Level.INFO, "Pending start of new searcher"); - } - - final String displayName = NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.displayName") - + (finalRun ? (" - " + NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.finalizeMsg")) : ""); - progressGroup = AggregateProgressFactory.createSystemHandle(displayName + (" (" - + NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.pendingMsg") + ")"), null, new Cancellable() { - @Override - public boolean cancel() { - logger.log(Level.INFO, "Cancelling the searcher by user."); - if (progressGroup != null) { - progressGroup.setDisplayName(displayName + " (" + NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.cancelMsg") + "...)"); - } - return Searcher.this.cancel(true); - } - }, null); - - updateKeywords(); - - ProgressContributor[] subProgresses = new ProgressContributor[keywords.size()]; - int i = 0; - for (Keyword keywordQuery : keywords) { - subProgresses[i] = - AggregateProgressFactory.createProgressContributor(keywordQuery.getQuery()); - progressGroup.addContributor(subProgresses[i]); - i++; - } - - progressGroup.start(); - - //block to ensure previous searcher is completely done with doInBackground() - //even after previous searcher cancellation, we need to check this - searcherLock.lock(); - final StopWatch stopWatch = new StopWatch(); - stopWatch.start(); - try { - logger.log(Level.INFO, "Started a new searcher"); - progressGroup.setDisplayName(displayName); - //make sure other searchers are not spawned - searcherDone = false; - runSearcher = false; - if (searchTimer.isRunning()) { - searchTimer.stop(); - } - - int keywordsSearched = 0; - - //updateKeywords(); - - for (Keyword keywordQuery : keywords) { - if (this.isCancelled()) { - logger.log(Level.INFO, "Cancel detected, bailing before new keyword processed: {0}", keywordQuery.getQuery()); - return null; - } - - final String queryStr = keywordQuery.getQuery(); - final KeywordList list = keywordToList.get(queryStr); - final String listName = list.getName(); - - //new subProgress will be active after the initial query - //when we know number of hits to start() with - if (keywordsSearched > 0) { - subProgresses[keywordsSearched - 1].finish(); - } - - - KeywordSearchQuery del = null; - - boolean isRegex = !keywordQuery.isLiteral(); - if (isRegex) { - del = new TermComponentQuery(keywordQuery); - } else { - del = new LuceneQuery(keywordQuery); - del.escape(); - } - - //limit search to currently ingested data sources - //set up a filter with 1 or more image ids OR'ed - final KeywordQueryFilter dataSourceFilter = new KeywordQueryFilter(KeywordQueryFilter.FilterType.DATA_SOURCE, curDataSourceIds); - del.addFilter(dataSourceFilter); - - Map> queryResult; - - try { - queryResult = del.performQuery(); - } catch (NoOpenCoreException ex) { - logger.log(Level.WARNING, "Error performing query: " + keywordQuery.getQuery(), ex); - //no reason to continue with next query if recovery failed - //or wait for recovery to kick in and run again later - //likely case has closed and threads are being interrupted - return null; - } catch (CancellationException e) { - logger.log(Level.INFO, "Cancel detected, bailing during keyword query: {0}", keywordQuery.getQuery()); - return null; - } catch (Exception e) { - logger.log(Level.WARNING, "Error performing query: " + keywordQuery.getQuery(), e); - continue; - } - - // calculate new results but substracting results already obtained in this ingest - // this creates a map of each keyword to the list of unique files that have that hit. - Map> newResults = filterResults(queryResult, isRegex); - - if (!newResults.isEmpty()) { - - //write results to BB - - //new artifacts created, to report to listeners - Collection newArtifacts = new ArrayList<>(); - - //scale progress bar more more granular, per result sub-progress, within per keyword - int totalUnits = newResults.size(); - subProgresses[keywordsSearched].start(totalUnits); - int unitProgress = 0; - String queryDisplayStr = keywordQuery.getQuery(); - if (queryDisplayStr.length() > 50) { - queryDisplayStr = queryDisplayStr.substring(0, 49) + "..."; - } - subProgresses[keywordsSearched].progress(listName + ": " + queryDisplayStr, unitProgress); - - - /* cycle through the keywords returned -- only one unless it was a regexp */ - for (final Keyword hitTerm : newResults.keySet()) { - //checking for cancellation between results - if (this.isCancelled()) { - logger.log(Level.INFO, "Cancel detected, bailing before new hit processed for query: {0}", keywordQuery.getQuery()); - return null; - } - - // update progress display - String hitDisplayStr = hitTerm.getQuery(); - if (hitDisplayStr.length() > 50) { - hitDisplayStr = hitDisplayStr.substring(0, 49) + "..."; - } - subProgresses[keywordsSearched].progress(listName + ": " + hitDisplayStr, unitProgress); - //subProgresses[keywordsSearched].progress(unitProgress); - - // this returns the unique files in the set with the first chunk that has a hit - Map contentHitsFlattened = ContentHit.flattenResults(newResults.get(hitTerm)); - for (final AbstractFile hitFile : contentHitsFlattened.keySet()) { - - // get the snippet for the first hit in the file - String snippet; - final String snippetQuery = KeywordSearchUtil.escapeLuceneQuery(hitTerm.getQuery()); - int chunkId = contentHitsFlattened.get(hitFile); - try { - snippet = LuceneQuery.querySnippet(snippetQuery, hitFile.getId(), chunkId, isRegex, true); - } catch (NoOpenCoreException e) { - logger.log(Level.WARNING, "Error querying snippet: " + snippetQuery, e); - //no reason to continue - return null; - } catch (Exception e) { - logger.log(Level.WARNING, "Error querying snippet: " + snippetQuery, e); - continue; - } - - // write the blackboard artifact for this keyword in this file - KeywordWriteResult written = del.writeToBlackBoard(hitTerm.getQuery(), hitFile, snippet, listName); - if (written == null) { - logger.log(Level.WARNING, "BB artifact for keyword hit not written, file: {0}, hit: {1}", new Object[]{hitFile, hitTerm.toString()}); - continue; - } - - newArtifacts.add(written.getArtifact()); - - //generate an ingest inbox message for this keyword in this file - if (list.getIngestMessages()) { - StringBuilder subjectSb = new StringBuilder(); - StringBuilder detailsSb = new StringBuilder(); - //final int hitFiles = newResults.size(); - - if (!keywordQuery.isLiteral()) { - subjectSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.regExpHitLbl")); - } else { - subjectSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.kwHitLbl")); - } - //subjectSb.append("<"); - String uniqueKey = null; - BlackboardAttribute attr = written.getAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD.getTypeID()); - if (attr != null) { - final String keyword = attr.getValueString(); - subjectSb.append(keyword); - uniqueKey = keyword.toLowerCase(); - } - - //subjectSb.append(">"); - //String uniqueKey = queryStr; - - //details - detailsSb.append(""); - //hit - detailsSb.append(""); - detailsSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.kwHitLThLbl")); - detailsSb.append(""); - detailsSb.append(""); - - //preview - attr = written.getAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_PREVIEW.getTypeID()); - if (attr != null) { - detailsSb.append(""); - detailsSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.previewThLbl")); - detailsSb.append(""); - detailsSb.append(""); - - } - - //file - detailsSb.append(""); - detailsSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.fileThLbl")); - detailsSb.append(""); - - detailsSb.append(""); - - - //list - attr = written.getAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME.getTypeID()); - detailsSb.append(""); - detailsSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.listThLbl")); - detailsSb.append(""); - detailsSb.append(""); - - //regex - if (!keywordQuery.isLiteral()) { - attr = written.getAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_REGEXP.getTypeID()); - if (attr != null) { - detailsSb.append(""); - detailsSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.regExThLbl")); - detailsSb.append(""); - detailsSb.append(""); - - } - } - detailsSb.append("
").append(EscapeUtil.escapeHtml(attr.getValueString())).append("
").append(EscapeUtil.escapeHtml(attr.getValueString())).append("
").append(hitFile.getParentPath()).append(hitFile.getName()).append("
").append(attr.getValueString()).append("
").append(attr.getValueString()).append("
"); - - services.postMessage(IngestMessage.createDataMessage(++messageID, KeywordSearchModuleFactory.getModuleName(), subjectSb.toString(), detailsSb.toString(), uniqueKey, written.getArtifact())); - } - } //for each file hit - - ++unitProgress; - - }//for each hit term - - //update artifact browser - if (!newArtifacts.isEmpty()) { - services.fireModuleDataEvent(new ModuleDataEvent(KeywordSearchModuleFactory.getModuleName(), ARTIFACT_TYPE.TSK_KEYWORD_HIT, newArtifacts)); - } - } //if has results - - //reset the status text before it goes away - subProgresses[keywordsSearched].progress(""); - - ++keywordsSearched; - - } //for each keyword - - } //end try block - catch (Exception ex) { - logger.log(Level.WARNING, "searcher exception occurred", ex); - } finally { - try { - finalizeSearcher(); - stopWatch.stop(); - logger.log(Level.INFO, "Searcher took to run: {0} secs.", stopWatch.getElapsedTimeSecs()); - } finally { - searcherLock.unlock(); - } - } - - return null; - } - - @Override - protected void done() { - // call get to see if there were any errors - try { - get(); - } catch (InterruptedException | ExecutionException e) { - logger.log(Level.SEVERE, "Error performing keyword search: " + e.getMessage()); - services.postMessage(IngestMessage.createErrorMessage(++messageID, KeywordSearchModuleFactory.getModuleName(), "Error performing keyword search", e.getMessage())); - } // catch and ignore if we were cancelled - catch (java.util.concurrent.CancellationException ex) { - } - } - - /** - * Sync-up the updated keywords from the currently used lists in the XML - */ - private void updateKeywords() { - KeywordSearchListsXML loader = KeywordSearchListsXML.getCurrent(); - - this.keywords.clear(); - this.keywordToList.clear(); - - for (String name : this.keywordLists) { - KeywordList list = loader.getList(name); - for (Keyword k : list.getKeywords()) { - this.keywords.add(k); - this.keywordToList.put(k.getQuery(), list); - } - } - - - } - - //perform all essential cleanup that needs to be done right AFTER doInBackground() returns - //without relying on done() method that is not guaranteed to run after background thread completes - //NEED to call this method always right before doInBackground() returns - /** - * Performs the cleanup that needs to be done right AFTER - * doInBackground() returns without relying on done() method that is not - * guaranteed to run after background thread completes REQUIRED to call - * this method always right before doInBackground() returns - */ - private void finalizeSearcher() { - logger.log(Level.INFO, "Searcher finalizing"); - SwingUtilities.invokeLater(new Runnable() { - @Override - public void run() { - progressGroup.finish(); - } - }); - searcherDone = true; //next currentSearcher can start - - if (finalRun) { - //this is the final searcher - logger.log(Level.INFO, "The final searcher in this ingest done."); - - //run module cleanup - cleanup(); - } else { - //start counting time for a new searcher to start - //unless final searcher is pending - if (finalSearcher == null) { - //we need a new Timer object, because restarting previus will not cause firing of the action - final int updateIntervalMs = KeywordSearchSettings.getUpdateFrequency().getTime() * 60 * 1000; - searchTimer = new Timer(updateIntervalMs, new SearchTimerAction()); - searchTimer.start(); - } - } - } - - //calculate new results but substracting results already obtained in this ingest - //update currentResults map with the new results - private Map> filterResults(Map> queryResult, boolean isRegex) { - Map> newResults = new HashMap<>(); - - for (String termResult : queryResult.keySet()) { - List queryTermResults = queryResult.get(termResult); - - //translate to list of IDs that we keep track of - List queryTermResultsIDs = new ArrayList<>(); - for (ContentHit ch : queryTermResults) { - queryTermResultsIDs.add(ch.getId()); - } - - Keyword termResultK = new Keyword(termResult, !isRegex); - List curTermResults = currentResults.get(termResultK); - if (curTermResults == null) { - currentResults.put(termResultK, queryTermResultsIDs); - newResults.put(termResultK, queryTermResults); - } else { - //some AbstractFile hits already exist for this keyword - for (ContentHit res : queryTermResults) { - if (!curTermResults.contains(res.getId())) { - //add to new results - List newResultsFs = newResults.get(termResultK); - if (newResultsFs == null) { - newResultsFs = new ArrayList<>(); - newResults.put(termResultK, newResultsFs); - } - newResultsFs.add(res); - curTermResults.add(res.getId()); - } - } - } - } - - return newResults; - - } - } } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchListsManagementPanel.form b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchListsManagementPanel.form index cb2629ae11..5576e62923 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchListsManagementPanel.form +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchListsManagementPanel.form @@ -3,7 +3,7 @@
- + diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchListsManagementPanel.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchListsManagementPanel.java index 83348f6bd3..41f6c7665a 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchListsManagementPanel.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchListsManagementPanel.java @@ -97,7 +97,7 @@ class KeywordSearchListsManagementPanel extends javax.swing.JPanel implements Op importButton = new javax.swing.JButton(); keywordListsLabel = new javax.swing.JLabel(); - setMinimumSize(new java.awt.Dimension(200, 0)); + setMinimumSize(new java.awt.Dimension(250, 0)); setPreferredSize(new java.awt.Dimension(250, 492)); jScrollPane1.setPreferredSize(new java.awt.Dimension(200, 402)); @@ -146,7 +146,7 @@ class KeywordSearchListsManagementPanel extends javax.swing.JPanel implements Op .addComponent(newListButton, javax.swing.GroupLayout.PREFERRED_SIZE, 114, javax.swing.GroupLayout.PREFERRED_SIZE) .addGap(18, 18, 18) .addComponent(importButton, javax.swing.GroupLayout.PREFERRED_SIZE, 126, javax.swing.GroupLayout.PREFERRED_SIZE))) - .addGap(0, 4, Short.MAX_VALUE))) + .addGap(0, 0, Short.MAX_VALUE))) .addContainerGap()) ); layout.setVerticalGroup( diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchListsViewerPanel.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchListsViewerPanel.java index de6d93a918..8464a1ecc5 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchListsViewerPanel.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchListsViewerPanel.java @@ -21,6 +21,7 @@ package org.sleuthkit.autopsy.keywordsearch; import java.awt.Component; import java.awt.Cursor; +import java.awt.EventQueue; import java.awt.event.ActionEvent; import java.awt.event.ActionListener; import java.beans.PropertyChangeEvent; @@ -114,26 +115,23 @@ class KeywordSearchListsViewerPanel extends AbstractKeywordSearchPerformer { } }); - if (IngestManager.getInstance().isIngestRunning()) { - initIngest(true); - } else { - initIngest(false); - } + ingestRunning = IngestManager.getInstance().isIngestRunning(); + updateComponents(); IngestManager.addPropertyChangeListener(new PropertyChangeListener() { @Override public void propertyChange(PropertyChangeEvent evt) { String changed = evt.getPropertyName(); - Object oldValue = evt.getOldValue(); - if (changed.equals(IngestEvent.COMPLETED.toString()) - && ((String) oldValue).equals(KeywordSearchModuleFactory.getModuleName())) { - initIngest(false); - } else if (changed.equals(IngestEvent.STARTED.toString()) - && ((String) oldValue).equals(KeywordSearchModuleFactory.getModuleName())) { - initIngest(true); - } else if (changed.equals(IngestEvent.STOPPED.toString()) - && ((String) oldValue).equals(KeywordSearchModuleFactory.getModuleName())) { - initIngest(false); + if (changed.equals(IngestEvent.INGEST_JOB_STARTED.toString()) + || changed.equals(IngestEvent.INGEST_JOB_COMPLETED.toString()) + || changed.equals(IngestEvent.INGEST_JOB_CANCELLED.toString())) { + EventQueue.invokeLater(new Runnable() { + @Override + public void run() { + ingestRunning = IngestManager.getInstance().isIngestRunning(); + updateComponents(); + } + }); } } }); @@ -141,7 +139,8 @@ class KeywordSearchListsViewerPanel extends AbstractKeywordSearchPerformer { searchAddListener = new ActionListener() { @Override public void actionPerformed(ActionEvent e) { - if (ingestRunning) { + if (ingestRunning) { + SearchRunner.getInstance().addKeywordListsToAllJobs(listsTableModel.getSelectedLists()); logger.log(Level.INFO, "Submitted enqueued lists to ingest"); } else { searchAction(e); @@ -152,28 +151,21 @@ class KeywordSearchListsViewerPanel extends AbstractKeywordSearchPerformer { searchAddButton.addActionListener(searchAddListener); } - /** - * Initialize this panel depending on whether ingest is running - * - * @param running case 0: ingest running case 1: ingest not running - */ - private void initIngest(boolean running) { - if (running) { - ingestRunning = true; + private void updateComponents() { + ingestRunning = IngestManager.getInstance().isIngestRunning(); + if (ingestRunning) { searchAddButton.setText(NbBundle.getMessage(this.getClass(), "KeywordSearchListsViewerPanel.initIngest.addIngestTitle")); searchAddButton.setToolTipText(NbBundle.getMessage(this.getClass(), "KeywordSearchListsViewerPanel.initIngest.addIngestMsg" )); - listsTableModel.resync(); } else { - ingestRunning = false; searchAddButton.setText(NbBundle.getMessage(this.getClass(), "KeywordSearchListsViewerPanel.initIngest.searchIngestTitle")); searchAddButton.setToolTipText(NbBundle.getMessage(this.getClass(), "KeywordSearchListsViewerPanel.initIngest.addIdxSearchMsg")); - listsTableModel.resync(); } - updateIngestIndexLabel(running); + listsTableModel.resync(); + updateIngestIndexLabel(); } - private void updateIngestIndexLabel(boolean ingestRunning) { + private void updateIngestIndexLabel() { if (ingestRunning) { ingestIndexLabel.setText(NbBundle.getMessage(this.getClass(), "KeywordSearchListsViewerPanel.initIngest.ongoingIngestMsg", filesIndexed)); } @@ -184,7 +176,7 @@ class KeywordSearchListsViewerPanel extends AbstractKeywordSearchPerformer { @Override protected void postFilesIndexedChange() { - updateIngestIndexLabel(ingestRunning); + updateIngestIndexLabel(); } /** diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchListsXML.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchListsXML.java index 6a6bd9fc87..929ec49ac3 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchListsXML.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchListsXML.java @@ -66,7 +66,7 @@ final class KeywordSearchListsXML extends KeywordSearchListsAbstract { * RJCTODO: Move this one to the manager * @return */ - static KeywordSearchListsXML getCurrent() { + static synchronized KeywordSearchListsXML getCurrent() { if (currentInstance == null) { currentInstance = new KeywordSearchListsXML(CUR_LISTS_FILE); currentInstance.reload(); diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java new file mode 100644 index 0000000000..813c0fd8dd --- /dev/null +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java @@ -0,0 +1,713 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2011 - 2014 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.sleuthkit.autopsy.keywordsearch; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.concurrent.CancellationException; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.atomic.AtomicLong; +import java.util.logging.Level; +import javax.swing.SwingUtilities; +import javax.swing.SwingWorker; +import java.util.Timer; +import java.util.TimerTask; +import org.netbeans.api.progress.aggregate.AggregateProgressFactory; +import org.netbeans.api.progress.aggregate.AggregateProgressHandle; +import org.netbeans.api.progress.aggregate.ProgressContributor; +import org.openide.util.Cancellable; +import org.openide.util.NbBundle; +import org.sleuthkit.autopsy.coreutils.EscapeUtil; +import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.autopsy.coreutils.StopWatch; +import org.sleuthkit.autopsy.ingest.IngestMessage; +import org.sleuthkit.autopsy.ingest.IngestServices; +import org.sleuthkit.autopsy.ingest.ModuleDataEvent; +import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.BlackboardArtifact; +import org.sleuthkit.datamodel.BlackboardAttribute; + +/** + * Singleton keyword search manager: + * Launches search threads for each job and performs commits, both on timed + * intervals. + */ +public final class SearchRunner { + private static final Logger logger = Logger.getLogger(SearchRunner.class.getName()); + private static SearchRunner instance = null; + private IngestServices services = IngestServices.getInstance(); + private Ingester ingester = null; //guarded by "ingester" + private volatile boolean updateTimerRunning = false; + private Timer updateTimer; + private Map jobs = new HashMap<>(); //guarded by "this" + + SearchRunner() { + ingester = Server.getIngester(); + updateTimer = new Timer("SearchRunner update timer", true); // run as a daemon + } + + /** + * + * @return the singleton object + */ + public static synchronized SearchRunner getInstance() { + if (instance == null) { + instance = new SearchRunner(); + } + return instance; + } + + /** + * + * @param jobId + * @param dataSourceId + * @param keywordListNames + */ + public synchronized void startJob(long jobId, long dataSourceId, List keywordListNames) { + if (!jobs.containsKey(jobId)) { + logger.log(Level.INFO, "Adding job {0}", jobId); + SearchJobInfo jobData = new SearchJobInfo(jobId, dataSourceId, keywordListNames); + jobs.put(jobId, jobData); + } + + jobs.get(jobId).incrementModuleReferenceCount(); + + if (jobs.size() > 0) { + final int updateIntervalMs = KeywordSearchSettings.getUpdateFrequency().getTime() * 60 * 1000; + if (!updateTimerRunning) { + updateTimer.scheduleAtFixedRate(new UpdateTimerTask(), updateIntervalMs, updateIntervalMs); + updateTimerRunning = true; + } + } + } + + /** + * Perform normal finishing of searching for this job, including one last + * commit and search. Blocks until the final search is complete. + * @param jobId + */ + public void endJob(long jobId) { + SearchJobInfo job; + boolean readyForFinalSearch = false; + synchronized(this) { + job = jobs.get(jobId); + if (job == null) { + return; + } + + // Only do final search if this is the last module in this job to call endJob() + if(job.decrementModuleReferenceCount() == 0) { + jobs.remove(jobId); + readyForFinalSearch = true; + } + } + + if (readyForFinalSearch) { + commit(); + doFinalSearch(job); //this will block until it's done + } + } + + + /** + * Immediate stop and removal of job from SearchRunner. Cancels the + * associated search worker if it's still running. + * @param jobId + */ + public void stopJob(long jobId) { + logger.log(Level.INFO, "Stopping job {0}", jobId); + commit(); + + SearchJobInfo job; + synchronized(this) { + job = jobs.get(jobId); + if (job == null) { + return; + } + + //stop currentSearcher + SearchRunner.Searcher currentSearcher = job.getCurrentSearcher(); + if ((currentSearcher != null) && (!currentSearcher.isDone())) { + currentSearcher.cancel(true); + } + + jobs.remove(jobId); + } + } + + /** + * Add these lists to all of the jobs + * @param keywordListName + */ + public synchronized void addKeywordListsToAllJobs(List keywordListNames) { + for(String listName : keywordListNames) { + logger.log(Level.INFO, "Adding keyword list {0} to all jobs", listName); + for(SearchJobInfo j : jobs.values()) { + j.addKeywordListName(listName); + } + } + } + + /** + * Commits index and notifies listeners of index update + */ + private void commit() { + synchronized(ingester) { + ingester.commit(); + } + + // Signal a potential change in number of text_ingested files + try { + final int numIndexedFiles = KeywordSearch.getServer().queryNumIndexedFiles(); + KeywordSearch.fireNumIndexedFilesChange(null, new Integer(numIndexedFiles)); + } catch (NoOpenCoreException | KeywordSearchModuleException ex) { + logger.log(Level.WARNING, "Error executing Solr query to check number of indexed files: ", ex); + } + } + + /** + * A final search waits for any still-running workers, and then executes a + * new one and waits until that is done. + * @param job + */ + private void doFinalSearch(SearchJobInfo job) { + // Run one last search as there are probably some new files committed + logger.log(Level.INFO, "Running final search for jobid {0}", job.getJobId()); + if (!job.getKeywordListNames().isEmpty()) { + try { + // In case this job still has a worker running, wait for it to finish + job.waitForCurrentWorker(); + + SearchRunner.Searcher finalSearcher = new SearchRunner.Searcher(job, true); + job.setCurrentSearcher(finalSearcher); //save the ref + finalSearcher.execute(); //start thread + + // block until the search is complete + finalSearcher.get(); + + } catch (InterruptedException | ExecutionException ex) { + logger.log(Level.WARNING, "Job {1} final search thread failed: {2}", new Object[]{job.getJobId(), ex}); + } + } + } + + + /** + * Timer triggered re-search for each job (does a single index commit first) + */ + private class UpdateTimerTask extends TimerTask { + private final Logger logger = Logger.getLogger(SearchRunner.UpdateTimerTask.class.getName()); + + @Override + public void run() { + // If no jobs then cancel the task. If more job(s) come along, a new task will start up. + if (jobs.isEmpty()) { + this.cancel(); //terminate this timer task + updateTimerRunning = false; + return; + } + + commit(); + + synchronized(SearchRunner.this) { + // Spawn a search thread for each job + for(Entry j : jobs.entrySet()) { + SearchJobInfo job = j.getValue(); + // If no lists or the worker is already running then skip it + if (!job.getKeywordListNames().isEmpty() && !job.isWorkerRunning()) { + Searcher searcher = new Searcher(job); + job.setCurrentSearcher(searcher); //save the ref + searcher.execute(); //start thread + job.setWorkerRunning(true); + } + } + } + } + } + + /** + * Data structure to keep track of keyword lists, current results, and search + * running status for each jobid + */ + private class SearchJobInfo { + private final long jobId; + private final long dataSourceId; + // mutable state: + private volatile boolean workerRunning; + private List keywordListNames; //guarded by SearchJobInfo.this + private Map> currentResults; //guarded by SearchJobInfo.this + private SearchRunner.Searcher currentSearcher; + private AtomicLong moduleReferenceCount = new AtomicLong(0); + private final Object finalSearchLock = new Object(); //used for a condition wait + + public SearchJobInfo(long jobId, long dataSourceId, List keywordListNames) { + this.jobId = jobId; + this.dataSourceId = dataSourceId; + this.keywordListNames = new ArrayList<>(keywordListNames); + currentResults = new HashMap<>(); + workerRunning = false; + currentSearcher = null; + } + + public long getJobId() { + return jobId; + } + + public long getDataSourceId() { + return dataSourceId; + } + + public synchronized List getKeywordListNames() { + return new ArrayList<>(keywordListNames); + } + + public synchronized void addKeywordListName(String keywordListName) { + if (!keywordListNames.contains(keywordListName)) { + keywordListNames.add(keywordListName); + } + } + + public synchronized List currentKeywordResults(Keyword k) { + return currentResults.get(k); + } + + public synchronized void addKeywordResults(Keyword k, List resultsIDs) { + currentResults.put(k, resultsIDs); + } + + public boolean isWorkerRunning() { + return workerRunning; + } + + public void setWorkerRunning(boolean flag) { + workerRunning = flag; + } + + public synchronized SearchRunner.Searcher getCurrentSearcher() { + return currentSearcher; + } + + public synchronized void setCurrentSearcher(SearchRunner.Searcher searchRunner) { + currentSearcher = searchRunner; + } + + public void incrementModuleReferenceCount() { + moduleReferenceCount.incrementAndGet(); + } + + public long decrementModuleReferenceCount() { + return moduleReferenceCount.decrementAndGet(); + } + + /** In case this job still has a worker running, wait for it to finish + * + * @throws InterruptedException + */ + public void waitForCurrentWorker() throws InterruptedException { + synchronized(finalSearchLock) { + while(workerRunning) { + finalSearchLock.wait(); //wait() releases the lock + } + } + } + + /** + * Unset workerRunning and wake up thread(s) waiting on finalSearchLock + */ + public void searchNotify() { + synchronized(finalSearchLock) { + workerRunning = false; + finalSearchLock.notify(); + } + } + } + + /** + * Searcher responsible for searching the current index and writing results + * to blackboard and the inbox. Also, posts results to listeners as Ingest + * data events. Searches entire index, and keeps track of only new results + * to report and save. Runs as a background thread. + */ + private final class Searcher extends SwingWorker { + + /** + * Searcher has private copies/snapshots of the lists and keywords + */ + private SearchJobInfo job; + private List keywords; //keywords to search + private List keywordListNames; // lists currently being searched + private Map keywordToList; //keyword to list name mapping + private AggregateProgressHandle progressGroup; + private final Logger logger = Logger.getLogger(SearchRunner.Searcher.class.getName()); + private boolean finalRun = false; + + Searcher(SearchJobInfo job) { + this.job = job; + keywordListNames = job.getKeywordListNames(); + keywords = new ArrayList<>(); + keywordToList = new HashMap<>(); + //keywords are populated as searcher runs + } + + Searcher(SearchJobInfo job, boolean finalRun) { + this(job); + this.finalRun = finalRun; + } + + @Override + protected Object doInBackground() throws Exception { + final String displayName = NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.displayName") + + (finalRun ? (" - " + NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.finalizeMsg")) : ""); + final String pgDisplayName = displayName + (" (" + NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.pendingMsg") + ")"); + progressGroup = AggregateProgressFactory.createSystemHandle(pgDisplayName, null, new Cancellable() { + @Override + public boolean cancel() { + logger.log(Level.INFO, "Cancelling the searcher by user."); + if (progressGroup != null) { + progressGroup.setDisplayName(displayName + " (" + NbBundle.getMessage(this.getClass(), "SearchRunner.doInBackGround.cancelMsg") + "...)"); + } + return SearchRunner.Searcher.this.cancel(true); + } + }, null); + + updateKeywords(); + + ProgressContributor[] subProgresses = new ProgressContributor[keywords.size()]; + int i = 0; + for (Keyword keywordQuery : keywords) { + subProgresses[i] = AggregateProgressFactory.createProgressContributor(keywordQuery.getQuery()); + progressGroup.addContributor(subProgresses[i]); + i++; + } + + progressGroup.start(); + + final StopWatch stopWatch = new StopWatch(); + stopWatch.start(); + try { + progressGroup.setDisplayName(displayName); + + int keywordsSearched = 0; + + for (Keyword keywordQuery : keywords) { + if (this.isCancelled()) { + logger.log(Level.INFO, "Cancel detected, bailing before new keyword processed: {0}", keywordQuery.getQuery()); + return null; + } + + final String queryStr = keywordQuery.getQuery(); + final KeywordList list = keywordToList.get(queryStr); + final String listName = list.getName(); + + //new subProgress will be active after the initial query + //when we know number of hits to start() with + if (keywordsSearched > 0) { + subProgresses[keywordsSearched - 1].finish(); + } + + KeywordSearchQuery del = null; + + boolean isRegex = !keywordQuery.isLiteral(); + if (isRegex) { + del = new TermComponentQuery(keywordQuery); + } else { + del = new LuceneQuery(keywordQuery); + del.escape(); + } + + //limit search to currently ingested data sources + //set up a filter with 1 or more image ids OR'ed + final KeywordQueryFilter dataSourceFilter = new KeywordQueryFilter(KeywordQueryFilter.FilterType.DATA_SOURCE, job.getDataSourceId()); + del.addFilter(dataSourceFilter); + + Map> queryResult; + + try { + queryResult = del.performQuery(); + } catch (NoOpenCoreException ex) { + logger.log(Level.WARNING, "Error performing query: " + keywordQuery.getQuery(), ex); + //no reason to continue with next query if recovery failed + //or wait for recovery to kick in and run again later + //likely case has closed and threads are being interrupted + return null; + } catch (CancellationException e) { + logger.log(Level.INFO, "Cancel detected, bailing during keyword query: {0}", keywordQuery.getQuery()); + return null; + } catch (Exception e) { + logger.log(Level.WARNING, "Error performing query: " + keywordQuery.getQuery(), e); + continue; + } + + // calculate new results by substracting results already obtained in this ingest + // this creates a map of each keyword to the list of unique files that have that hit. + Map> newResults = filterResults(queryResult, isRegex); + + if (!newResults.isEmpty()) { + + //write results to BB + + //new artifacts created, to report to listeners + Collection newArtifacts = new ArrayList<>(); + + //scale progress bar more more granular, per result sub-progress, within per keyword + int totalUnits = newResults.size(); + subProgresses[keywordsSearched].start(totalUnits); + int unitProgress = 0; + String queryDisplayStr = keywordQuery.getQuery(); + if (queryDisplayStr.length() > 50) { + queryDisplayStr = queryDisplayStr.substring(0, 49) + "..."; + } + subProgresses[keywordsSearched].progress(listName + ": " + queryDisplayStr, unitProgress); + + // cycle through the keywords returned -- only one unless it was a regexp + for (final Keyword hitTerm : newResults.keySet()) { + //checking for cancellation between results + if (this.isCancelled()) { + logger.log(Level.INFO, "Cancel detected, bailing before new hit processed for query: {0}", keywordQuery.getQuery()); + return null; + } + + // update progress display + String hitDisplayStr = hitTerm.getQuery(); + if (hitDisplayStr.length() > 50) { + hitDisplayStr = hitDisplayStr.substring(0, 49) + "..."; + } + subProgresses[keywordsSearched].progress(listName + ": " + hitDisplayStr, unitProgress); + + // this returns the unique files in the set with the first chunk that has a hit + Map contentHitsFlattened = ContentHit.flattenResults(newResults.get(hitTerm)); + for (final AbstractFile hitFile : contentHitsFlattened.keySet()) { + + // get the snippet for the first hit in the file + String snippet; + final String snippetQuery = KeywordSearchUtil.escapeLuceneQuery(hitTerm.getQuery()); + int chunkId = contentHitsFlattened.get(hitFile); + try { + snippet = LuceneQuery.querySnippet(snippetQuery, hitFile.getId(), chunkId, isRegex, true); + } catch (NoOpenCoreException e) { + logger.log(Level.WARNING, "Error querying snippet: " + snippetQuery, e); + //no reason to continue + return null; + } catch (Exception e) { + logger.log(Level.WARNING, "Error querying snippet: " + snippetQuery, e); + continue; + } + + // write the blackboard artifact for this keyword in this file + KeywordWriteResult written = del.writeToBlackBoard(hitTerm.getQuery(), hitFile, snippet, listName); + if (written == null) { + logger.log(Level.WARNING, "BB artifact for keyword hit not written, file: {0}, hit: {1}", new Object[]{hitFile, hitTerm.toString()}); + continue; + } + + newArtifacts.add(written.getArtifact()); + + //generate an ingest inbox message for this keyword in this file + if (list.getIngestMessages()) { + StringBuilder subjectSb = new StringBuilder(); + StringBuilder detailsSb = new StringBuilder(); + + if (!keywordQuery.isLiteral()) { + subjectSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.regExpHitLbl")); + } else { + subjectSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.kwHitLbl")); + } + String uniqueKey = null; + BlackboardAttribute attr = written.getAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD.getTypeID()); + if (attr != null) { + final String keyword = attr.getValueString(); + subjectSb.append(keyword); + uniqueKey = keyword.toLowerCase(); + } + + //details + detailsSb.append(""); + //hit + detailsSb.append(""); + detailsSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.kwHitThLbl")); + detailsSb.append(""); + detailsSb.append(""); + + //preview + attr = written.getAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_PREVIEW.getTypeID()); + if (attr != null) { + detailsSb.append(""); + detailsSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.previewThLbl")); + detailsSb.append(""); + detailsSb.append(""); + } + + //file + detailsSb.append(""); + detailsSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.fileThLbl")); + detailsSb.append(""); + detailsSb.append(""); + + //list + attr = written.getAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME.getTypeID()); + detailsSb.append(""); + detailsSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.listThLbl")); + detailsSb.append(""); + detailsSb.append(""); + + //regex + if (!keywordQuery.isLiteral()) { + attr = written.getAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_REGEXP.getTypeID()); + if (attr != null) { + detailsSb.append(""); + detailsSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.regExThLbl")); + detailsSb.append(""); + detailsSb.append(""); + } + } + detailsSb.append("
").append(EscapeUtil.escapeHtml(attr.getValueString())).append("
").append(EscapeUtil.escapeHtml(attr.getValueString())).append("
").append(hitFile.getParentPath()).append(hitFile.getName()).append("
").append(attr.getValueString()).append("
").append(attr.getValueString()).append("
"); + + services.postMessage(IngestMessage.createDataMessage(KeywordSearchModuleFactory.getModuleName(), subjectSb.toString(), detailsSb.toString(), uniqueKey, written.getArtifact())); + } + } //for each file hit + + ++unitProgress; + + }//for each hit term + + //update artifact browser + if (!newArtifacts.isEmpty()) { + services.fireModuleDataEvent(new ModuleDataEvent(KeywordSearchModuleFactory.getModuleName(), BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT, newArtifacts)); + } + } //if has results + + //reset the status text before it goes away + subProgresses[keywordsSearched].progress(""); + + ++keywordsSearched; + + } //for each keyword + + } //end try block + catch (Exception ex) { + logger.log(Level.WARNING, "searcher exception occurred", ex); + } finally { + try { + finalizeSearcher(); + stopWatch.stop(); + + logger.log(Level.INFO, "Searcher took to run: {0} secs.", stopWatch.getElapsedTimeSecs()); + } finally { + // In case a thread is waiting on this worker to be done + job.searchNotify(); + } + } + + return null; + } + + @Override + protected void done() { + // call get to see if there were any errors + try { + get(); + } catch (InterruptedException | ExecutionException e) { + logger.log(Level.SEVERE, "Error performing keyword search: " + e.getMessage()); + services.postMessage(IngestMessage.createErrorMessage(KeywordSearchModuleFactory.getModuleName(), "Error performing keyword search", e.getMessage())); + } // catch and ignore if we were cancelled + catch (java.util.concurrent.CancellationException ex) { + } + } + + /** + * Sync-up the updated keywords from the currently used lists in the XML + */ + private void updateKeywords() { + KeywordSearchListsXML loader = KeywordSearchListsXML.getCurrent(); + + keywords.clear(); + keywordToList.clear(); + + + for (String name : keywordListNames) { + KeywordList list = loader.getList(name); + for (Keyword k : list.getKeywords()) { + this.keywords.add(k); + this.keywordToList.put(k.getQuery(), list); + } + } + } + + /** + * Performs the cleanup that needs to be done right AFTER + * doInBackground() returns without relying on done() method that is not + * guaranteed to run. + */ + private void finalizeSearcher() { + SwingUtilities.invokeLater(new Runnable() { + @Override + public void run() { + progressGroup.finish(); + } + }); + } + + //calculate new results but substracting results already obtained in this ingest + //update currentResults map with the new results + private Map> filterResults(Map> queryResult, boolean isRegex) { + Map> newResults = new HashMap<>(); + + for (String termResult : queryResult.keySet()) { + List queryTermResults = queryResult.get(termResult); + + //translate to list of IDs that we keep track of + List queryTermResultsIDs = new ArrayList<>(); + for (ContentHit ch : queryTermResults) { + queryTermResultsIDs.add(ch.getId()); + } + + Keyword termResultK = new Keyword(termResult, !isRegex); + List curTermResults = job.currentKeywordResults(termResultK); + if (curTermResults == null) { + job.addKeywordResults(termResultK, queryTermResultsIDs); + newResults.put(termResultK, queryTermResults); + } else { + //some AbstractFile hits already exist for this keyword + for (ContentHit res : queryTermResults) { + if (!curTermResults.contains(res.getId())) { + //add to new results + List newResultsFs = newResults.get(termResultK); + if (newResultsFs == null) { + newResultsFs = new ArrayList<>(); + newResults.put(termResultK, newResultsFs); + } + newResultsFs.add(res); + curTermResults.add(res.getId()); + } + } + } + } + + return newResults; + + } + + } + +} diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java index 676ff39ee8..5b3baa041f 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java @@ -385,9 +385,9 @@ public class Server { logger.log(Level.INFO, "Finished starting Solr"); try { - //block, give time to fully start the process + //block for 10 seconds, give time to fully start the process //so if it's restarted solr operations can be resumed seamlessly - Thread.sleep(10000); + Thread.sleep(10 * 1000); } catch (InterruptedException ex) { logger.log(Level.WARNING, "Timer interrupted"); } @@ -398,19 +398,17 @@ public class Server { final List pids = this.getSolrPIDs(); logger.log(Level.INFO, "New Solr process PID: " + pids); - - } catch (SecurityException ex) { - logger.log(Level.WARNING, "Could not start Solr process!", ex); + logger.log(Level.SEVERE, "Could not start Solr process!", ex); throw new KeywordSearchModuleException( NbBundle.getMessage(this.getClass(), "Server.start.exception.cantStartSolr.msg"), ex); } catch (IOException ex) { - logger.log(Level.WARNING, "Could not start Solr server process!", ex); + logger.log(Level.SEVERE, "Could not start Solr server process!", ex); throw new KeywordSearchModuleException( NbBundle.getMessage(this.getClass(), "Server.start.exception.cantStartSolr.msg2"), ex); } } else { - logger.log(Level.WARNING, "Could not start Solr server process, port [" + currentSolrServerPort + "] not available!"); + logger.log(Level.SEVERE, "Could not start Solr server process, port [" + currentSolrServerPort + "] not available!"); throw new SolrServerNoPortException(currentSolrServerPort); } } diff --git a/RecentActivity/release/rr-full/plugins/lsasecrets.pl b/RecentActivity/release/rr-full/plugins/lsasecrets.pl index 399ba6519e..c4b195432e 100755 --- a/RecentActivity/release/rr-full/plugins/lsasecrets.pl +++ b/RecentActivity/release/rr-full/plugins/lsasecrets.pl @@ -49,23 +49,30 @@ sub pluginmain { eval { ::rptMsg(""); ::rptMsg("Domain secret - \$MACHINE\.ACC"); - my $c = $key->get_subkey("\$MACHINE\.ACC\\CupdTime")->get_value("")->get_data(); - my @v = unpack("VV",$c); - my $cupd = gmtime(::getTime($v[0],$v[1])); - ::rptMsg("CupdTime = ".$cupd); + my $v1 = $key->get_subkey("\$MACHINE\.ACC\\CupdTime")->get_value(""); + if (defined $v1) { + my $c = $v1->get_data(); + my @v = unpack("VV",$c); + my $cupd = gmtime(::getTime($v[0],$v[1])); + ::rptMsg("CupdTime = ".$cupd); + } + else { + ::rptMsg("CupdTime value not found"); + } - my $o = $key->get_subkey("\$MACHINE\.ACC\\OupdTime")->get_value("")->get_data(); - my @v = unpack("VV",$c); - my $oupd = gmtime(::getTime($v[0],$v[1])); - ::rptMsg("OupdTime = ".$oupd); + + $v1 = $key->get_subkey("\$MACHINE\.ACC\\OupdTime")->get_value("") + if (defined $v1) { + my $c = $v1->get_data(); + my @v = unpack("VV",$c); + my $oupd = gmtime(::getTime($v[0],$v[1])); + ::rptMsg("OupdTime = ".$oupd); + } + else { + ::rptMsg("OupdTime value not found"); + } }; ::rptMsg("Error: ".$@) if ($@); - - - - - - } else { ::rptMsg($key_path." not found."); diff --git a/RecentActivity/release/rr-full/plugins/productpolicy.pl b/RecentActivity/release/rr-full/plugins/productpolicy.pl index 80948e1ff2..daf4b96acf 100755 --- a/RecentActivity/release/rr-full/plugins/productpolicy.pl +++ b/RecentActivity/release/rr-full/plugins/productpolicy.pl @@ -69,29 +69,32 @@ sub pluginmain { my $key; my $key_path = "ControlSet00".$curr."\\Control\\ProductOptions"; if ($key = $root_key->get_subkey($key_path)) { - my $prod; eval { - $prod = $key->get_value("ProductPolicy")->get_data(); + my $v1 = $key->get_value("ProductPolicy"); + if (defined $v1) { + my $prod = $v1->get_data(); + my %pol = parseData($prod); + ::rptMsg(""); + ::rptMsg("Note: This plugin applies to Vista and Windows 2008 ONLY."); + ::rptMsg("For a listing of names and values, see:"); + ::rptMsg("http://www.geoffchappell.com/viewer.htm?doc=notes/windows/license/install.htm&tx=3,5,6;4"); + ::rptMsg(""); + foreach my $p (sort keys %pol) { + ::rptMsg($p." - ".$pol{$p}); + } + + if (exists $prodinfo{$pol{"Kernel\-ProductInfo"}}) { + ::rptMsg(""); + ::rptMsg("Kernel\-ProductInfo = ".$prodinfo{$pol{"Kernel\-ProductInfo"}}); + } + } + else { + ::rptMsg("Error getting ProductPolicy value"); + } }; if ($@) { ::rptMsg("Error getting ProductPolicy value: $@"); } - else { - my %pol = parseData($prod); - ::rptMsg(""); - ::rptMsg("Note: This plugin applies to Vista and Windows 2008 ONLY."); - ::rptMsg("For a listing of names and values, see:"); - ::rptMsg("http://www.geoffchappell.com/viewer.htm?doc=notes/windows/license/install.htm&tx=3,5,6;4"); - ::rptMsg(""); - foreach my $p (sort keys %pol) { - ::rptMsg($p." - ".$pol{$p}); - } - - if (exists $prodinfo{$pol{"Kernel\-ProductInfo"}}) { - ::rptMsg(""); - ::rptMsg("Kernel\-ProductInfo = ".$prodinfo{$pol{"Kernel\-ProductInfo"}}); - } - } } else { ::rptMsg($key_path." not found."); diff --git a/RecentActivity/release/rr-full/plugins/skype.pl b/RecentActivity/release/rr-full/plugins/skype.pl index 3c83bc65f1..aac6de219c 100755 --- a/RecentActivity/release/rr-full/plugins/skype.pl +++ b/RecentActivity/release/rr-full/plugins/skype.pl @@ -45,13 +45,17 @@ sub pluginmain { ::rptMsg("LastWrite Time ".gmtime($key->get_timestamp())." (UTC)"); ::rptMsg(""); - my $install; eval { - $install = $key->get_subkey("Installer")->get_value("DonwloadLastModified")->get_data(); - ::rptMsg("DonwloadLastModified = ".$install); + my $v1 = $install = $key->get_subkey("Installer")->get_value("DonwloadLastModified"); + if (defined $v1) { + my $install = $v1->get_data() + ::rptMsg("DonwloadLastModified = ".$install); + } + else { + ::rptMsg("DonwloadLastModified value not found"); + } }; ::rptMsg("DonwloadLastModified value not found: ".$@) if ($@); - } else { ::rptMsg($key_path." not found."); diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Chrome.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Chrome.java index 8554fe04a7..b0b8cae007 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Chrome.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Chrome.java @@ -39,7 +39,7 @@ import java.io.FileReader; import java.io.IOException; import org.sleuthkit.autopsy.casemodule.services.FileManager; import org.sleuthkit.autopsy.coreutils.Logger; -import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleStatusHelper; +import org.sleuthkit.autopsy.ingest.IngestJobContext; import org.sleuthkit.autopsy.ingest.ModuleDataEvent; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.BlackboardArtifact; @@ -62,27 +62,29 @@ class Chrome extends Extract { private static final String downloadQueryVersion30 = "SELECT current_path as full_path, url, start_time, received_bytes FROM downloads, downloads_url_chains WHERE downloads.id=downloads_url_chains.id"; private static final String loginQuery = "select origin_url, username_value, signon_realm from logins"; private final Logger logger = Logger.getLogger(this.getClass().getName()); + private Content dataSource; + private IngestJobContext context; Chrome() { moduleName = NbBundle.getMessage(Chrome.class, "Chrome.moduleName"); } @Override - public void process(Content dataSource, DataSourceIngestModuleStatusHelper statusHelper) { + public void process(Content dataSource, IngestJobContext context) { + this.dataSource = dataSource; + this.context = context; dataFound = false; - this.getHistory(dataSource, statusHelper); - this.getBookmark(dataSource, statusHelper); - this.getCookie(dataSource, statusHelper); - this.getLogin(dataSource, statusHelper); - this.getDownload(dataSource, statusHelper); + this.getHistory(); + this.getBookmark(); + this.getCookie(); + this.getLogin(); + this.getDownload(); } /** * Query for history databases and add artifacts - * @param dataSource - * @param controller */ - private void getHistory(Content dataSource, DataSourceIngestModuleStatusHelper controller) { + private void getHistory() { FileManager fileManager = currentCase.getServices().getFileManager(); List historyFiles; try { @@ -126,7 +128,7 @@ class Chrome extends Extract { continue; } File dbFile = new File(temps); - if (controller.isIngestJobCancelled()) { + if (context.isJobCancelled()) { dbFile.delete(); break; } @@ -164,10 +166,8 @@ class Chrome extends Extract { /** * Search for bookmark files and make artifacts. - * @param dataSource - * @param controller */ - private void getBookmark(Content dataSource, DataSourceIngestModuleStatusHelper controller) { + private void getBookmark() { FileManager fileManager = currentCase.getServices().getFileManager(); List bookmarkFiles = null; try { @@ -204,7 +204,7 @@ class Chrome extends Extract { logger.log(Level.INFO, "{0}- Now getting Bookmarks from {1}", new Object[]{moduleName, temps}); File dbFile = new File(temps); - if (controller.isIngestJobCancelled()) { + if (context.isJobCancelled()) { dbFile.delete(); break; } @@ -305,10 +305,8 @@ class Chrome extends Extract { /** * Queries for cookie files and adds artifacts - * @param dataSource - * @param controller */ - private void getCookie(Content dataSource, DataSourceIngestModuleStatusHelper controller) { + private void getCookie() { FileManager fileManager = currentCase.getServices().getFileManager(); List cookiesFiles; @@ -344,7 +342,7 @@ class Chrome extends Extract { continue; } File dbFile = new File(temps); - if (controller.isIngestJobCancelled()) { + if (context.isJobCancelled()) { dbFile.delete(); break; } @@ -383,10 +381,8 @@ class Chrome extends Extract { /** * Queries for download files and adds artifacts - * @param dataSource - * @param controller */ - private void getDownload(Content dataSource, DataSourceIngestModuleStatusHelper controller) { + private void getDownload() { FileManager fileManager = currentCase.getServices().getFileManager(); List downloadFiles = null; try { @@ -420,7 +416,7 @@ class Chrome extends Extract { continue; } File dbFile = new File(temps); - if (controller.isIngestJobCancelled()) { + if (context.isJobCancelled()) { dbFile.delete(); break; } @@ -473,10 +469,8 @@ class Chrome extends Extract { /** * Queries for login files and adds artifacts - * @param dataSource - * @param controller */ - private void getLogin(Content dataSource, DataSourceIngestModuleStatusHelper controller) { + private void getLogin() { FileManager fileManager = currentCase.getServices().getFileManager(); List signonFiles; try { @@ -511,7 +505,7 @@ class Chrome extends Extract { continue; } File dbFile = new File(temps); - if (controller.isIngestJobCancelled()) { + if (context.isJobCancelled()) { dbFile.delete(); break; } diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Extract.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Extract.java index a312d81739..7a22ac6a8c 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Extract.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Extract.java @@ -32,7 +32,7 @@ import java.util.logging.Level; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.casemodule.Case; -import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleStatusHelper; +import org.sleuthkit.autopsy.ingest.IngestJobContext; import org.sleuthkit.autopsy.ingest.IngestModule.IngestModuleException; import org.sleuthkit.datamodel.*; @@ -40,7 +40,7 @@ abstract class Extract { protected Case currentCase = Case.getCurrentCase(); protected SleuthkitCase tskCase = currentCase.getSleuthkitCase(); - public final Logger logger = Logger.getLogger(this.getClass().getName()); + private final Logger logger = Logger.getLogger(this.getClass().getName()); private final ArrayList errorMessages = new ArrayList<>(); String moduleName = ""; boolean dataFound = false; @@ -51,7 +51,7 @@ abstract class Extract { void init() throws IngestModuleException { } - abstract void process(Content dataSource, DataSourceIngestModuleStatusHelper controller); + abstract void process(Content dataSource, IngestJobContext context); void complete() { } diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractIE.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractIE.java index b1168dba98..21875852ee 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractIE.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractIE.java @@ -52,7 +52,7 @@ import org.sleuthkit.datamodel.BlackboardAttribute; import org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE; import org.sleuthkit.datamodel.Content; import org.sleuthkit.autopsy.coreutils.PlatformUtil; -import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleStatusHelper; +import org.sleuthkit.autopsy.ingest.IngestJobContext; import org.sleuthkit.datamodel.*; /** @@ -66,6 +66,8 @@ class ExtractIE extends Extract { private String JAVA_PATH; private static final SimpleDateFormat dateFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); private ExecUtil execPasco; + private Content dataSource; + private IngestJobContext context; ExtractIE() { moduleName = NbBundle.getMessage(ExtractIE.class, "ExtractIE.moduleName.text"); @@ -74,19 +76,19 @@ class ExtractIE extends Extract { } @Override - public void process(Content dataSource, DataSourceIngestModuleStatusHelper controller) { + public void process(Content dataSource, IngestJobContext context) { + this.dataSource = dataSource; + this.context = context; dataFound = false; - this.getBookmark(dataSource, controller); - this.getCookie(dataSource, controller); - this.getHistory(dataSource, controller); + this.getBookmark(); + this.getCookie(); + this.getHistory(); } /** * Finds the files storing bookmarks and creates artifacts - * @param dataSource - * @param controller */ - private void getBookmark(Content dataSource, DataSourceIngestModuleStatusHelper controller) { + private void getBookmark() { org.sleuthkit.autopsy.casemodule.services.FileManager fileManager = currentCase.getServices().getFileManager(); List favoritesFiles; try { @@ -110,7 +112,7 @@ class ExtractIE extends Extract { continue; } - if (controller.isIngestJobCancelled()) { + if (context.isJobCancelled()) { break; } @@ -180,10 +182,8 @@ class ExtractIE extends Extract { /** * Finds files that store cookies and adds artifacts for them. - * @param dataSource - * @param controller */ - private void getCookie(Content dataSource, DataSourceIngestModuleStatusHelper controller) { + private void getCookie() { org.sleuthkit.autopsy.casemodule.services.FileManager fileManager = currentCase.getServices().getFileManager(); List cookiesFiles; try { @@ -202,7 +202,7 @@ class ExtractIE extends Extract { dataFound = true; for (AbstractFile cookiesFile : cookiesFiles) { - if (controller.isIngestJobCancelled()) { + if (context.isJobCancelled()) { break; } if (cookiesFile.getSize() == 0) { @@ -257,11 +257,9 @@ class ExtractIE extends Extract { /** * Locates index.dat files, runs Pasco on them, and creates artifacts. - * @param dataSource - * @param controller */ - private void getHistory(Content dataSource, DataSourceIngestModuleStatusHelper controller) { - logger.log(Level.INFO, "Pasco results path: " + moduleTempResultsDir); + private void getHistory() { + logger.log(Level.INFO, "Pasco results path: {0}", moduleTempResultsDir); boolean foundHistory = false; final File pascoRoot = InstalledFileLocator.getDefault().locate("pasco2", ExtractIE.class.getPackage().getName(), false); @@ -273,7 +271,7 @@ class ExtractIE extends Extract { } final String pascoHome = pascoRoot.getAbsolutePath(); - logger.log(Level.INFO, "Pasco2 home: " + pascoHome); + logger.log(Level.INFO, "Pasco2 home: {0}", pascoHome); PASCO_LIB_PATH = pascoHome + File.separator + "pasco2.jar" + File.pathSeparator + pascoHome + File.separator + "*"; @@ -283,7 +281,7 @@ class ExtractIE extends Extract { // get index.dat files org.sleuthkit.autopsy.casemodule.services.FileManager fileManager = currentCase.getServices().getFileManager(); - List indexFiles = null; + List indexFiles; try { indexFiles = fileManager.findFiles(dataSource, "index.dat"); } catch (TskCoreException ex) { @@ -312,7 +310,7 @@ class ExtractIE extends Extract { //indexFileName = "index" + Long.toString(bbart.getArtifactID()) + ".dat"; temps = RAImageIngestModule.getRATempPath(currentCase, "IE") + File.separator + indexFileName; File datFile = new File(temps); - if (controller.isIngestJobCancelled()) { + if (context.isJobCancelled()) { break; } try { @@ -337,7 +335,7 @@ class ExtractIE extends Extract { //Delete index.dat file since it was succcessfully by Pasco datFile.delete(); } else { - logger.log(Level.WARNING, "pasco execution failed on: " + this.getName()); + logger.log(Level.WARNING, "pasco execution failed on: {0}", this.getName()); this.addErrorMessage( NbBundle.getMessage(this.getClass(), "ExtractIE.getHistory.errMsg.errProcHist", this.getName())); } @@ -361,7 +359,7 @@ class ExtractIE extends Extract { Writer writer = null; try { final String outputFileFullPath = moduleTempResultsDir + File.separator + outputFileName; - logger.log(Level.INFO, "Writing pasco results to: " + outputFileFullPath); + logger.log(Level.INFO, "Writing pasco results to: {0}", outputFileFullPath); writer = new FileWriter(outputFileFullPath); execPasco = new ExecUtil(); execPasco.execute(writer, JAVA_PATH, @@ -402,7 +400,7 @@ class ExtractIE extends Extract { this.addErrorMessage( NbBundle.getMessage(this.getClass(), "ExtractIE.parsePascoOutput.errMsg.notFound", this.getName(), file.getName())); - logger.log(Level.WARNING, "Pasco Output not found: " + file.getPath()); + logger.log(Level.WARNING, "Pasco Output not found: {0}", file.getPath()); return; } diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java index 5a75e9fbca..2e3cadcde9 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java @@ -37,8 +37,8 @@ import org.sleuthkit.autopsy.coreutils.ExecUtil; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.PlatformUtil; import org.sleuthkit.autopsy.datamodel.ContentUtils; -import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleStatusHelper; -import org.sleuthkit.autopsy.recentactivity.ExtractUSB.USBInfo; +import org.sleuthkit.autopsy.ingest.IngestJobContext; +import org.sleuthkit.autopsy.recentactivity.UsbDeviceIdMapper.USBInfo; import org.sleuthkit.datamodel.*; import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE; import org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE; @@ -64,6 +64,8 @@ class ExtractRegistry extends Extract { private boolean rrFullFound = false; // true if we found the full version of regripper final private static String MODULE_VERSION = "1.0"; private ExecUtil execRR; + private Content dataSource; + private IngestJobContext context; //hide public constructor to prevent from instantiation by ingest module loader ExtractRegistry() { @@ -77,7 +79,7 @@ class ExtractRegistry extends Extract { } final String rrHome = rrRoot.getAbsolutePath(); - logger.log(Level.INFO, "RegRipper home: " + rrHome); + logger.log(Level.INFO, "RegRipper home: {0}", rrHome); if (PlatformUtil.isWindowsOS()) { RR_PATH = rrHome + File.separator + "rip.exe"; @@ -94,7 +96,7 @@ class ExtractRegistry extends Extract { } final String rrFullHome = rrFullRoot.getAbsolutePath(); - logger.log(Level.INFO, "RegRipper Full home: " + rrFullHome); + logger.log(Level.INFO, "RegRipper Full home: {0}", rrFullHome); if (PlatformUtil.isWindowsOS()) { RR_FULL_PATH = rrFullHome + File.separator + "rip.exe"; @@ -105,10 +107,8 @@ class ExtractRegistry extends Extract { /** * Search for the registry hives on the system. - * @param dataSource Data source to search for hives in. - * @return List of registry hives */ - private List findRegistryFiles(Content dataSource) { + private List findRegistryFiles() { List allRegistryFiles = new ArrayList<>(); org.sleuthkit.autopsy.casemodule.services.FileManager fileManager = currentCase.getServices().getFileManager(); @@ -138,12 +138,9 @@ class ExtractRegistry extends Extract { /** * Identifies registry files in the database by mtimeItem, runs regripper on them, and parses the output. - * - * @param dataSource - * @param controller */ - private void analyzeRegistryFiles(Content dataSource, DataSourceIngestModuleStatusHelper controller) { - List allRegistryFiles = findRegistryFiles(dataSource); + private void analyzeRegistryFiles() { + List allRegistryFiles = findRegistryFiles(); // open the log file FileWriter logFile = null; @@ -153,7 +150,7 @@ class ExtractRegistry extends Extract { java.util.logging.Logger.getLogger(ExtractRegistry.class.getName()).log(Level.SEVERE, null, ex); } - ExtractUSB extrctr = new ExtractUSB(); + UsbDeviceIdMapper usbMapper = new UsbDeviceIdMapper(); int j = 0; for (AbstractFile regFile : allRegistryFiles) { @@ -171,7 +168,7 @@ class ExtractRegistry extends Extract { continue; } - if (controller.isIngestJobCancelled()) { + if (context.isJobCancelled()) { break; } @@ -187,13 +184,13 @@ class ExtractRegistry extends Extract { logger.log(Level.INFO, moduleName + "- Now getting registry information from " + regFileNameLocal); RegOutputFiles regOutputFiles = executeRegRip(regFileNameLocal, outputPathBase); - if (controller.isIngestJobCancelled()) { + if (context.isJobCancelled()) { break; } // parse the autopsy-specific output if (regOutputFiles.autopsyPlugins.isEmpty() == false) { - if (parseAutopsyPluginOutput(regOutputFiles.autopsyPlugins, regFile.getId(), extrctr) == false) { + if (parseAutopsyPluginOutput(regOutputFiles.autopsyPlugins, regFile.getId(), usbMapper) == false) { this.addErrorMessage( NbBundle.getMessage(this.getClass(), "ExtractRegistry.analyzeRegFiles.failedParsingResults", this.getName(), regFileName)); @@ -365,7 +362,7 @@ class ExtractRegistry extends Extract { } // @@@ VERIFY that we are doing the right thing when we parse multiple NTUSER.DAT - private boolean parseAutopsyPluginOutput(String regRecord, long orgId, ExtractUSB extrctr) { + private boolean parseAutopsyPluginOutput(String regRecord, long orgId, UsbDeviceIdMapper extrctr) { FileInputStream fstream = null; try { SleuthkitCase tempDb = currentCase.getSleuthkitCase(); @@ -448,7 +445,7 @@ class ExtractRegistry extends Extract { String dev = artnode.getAttribute("dev"); String model = dev; if (dev.toLowerCase().contains("vid")) { - USBInfo info = extrctr.get(dev); + USBInfo info = extrctr.parseAndLookup(dev); if(info.getVendor()!=null) bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DEVICE_MAKE.getTypeID(), NbBundle.getMessage(this.getClass(), @@ -572,8 +569,10 @@ class ExtractRegistry extends Extract { } @Override - public void process(Content dataSource, DataSourceIngestModuleStatusHelper controller) { - analyzeRegistryFiles(dataSource, controller); + public void process(Content dataSource, IngestJobContext context) { + this.dataSource = dataSource; + this.context = context; + analyzeRegistryFiles(); } @Override diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Firefox.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Firefox.java index 9cfd1f30ef..1208d96080 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Firefox.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Firefox.java @@ -36,7 +36,7 @@ import org.openide.util.NbBundle; import org.sleuthkit.autopsy.casemodule.services.FileManager; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.datamodel.ContentUtils; -import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleStatusHelper; +import org.sleuthkit.autopsy.ingest.IngestJobContext; import org.sleuthkit.autopsy.ingest.IngestServices; import org.sleuthkit.autopsy.ingest.ModuleDataEvent; import org.sleuthkit.datamodel.AbstractFile; @@ -52,6 +52,7 @@ import org.sleuthkit.datamodel.TskCoreException; */ class Firefox extends Extract { + private static final Logger logger = Logger.getLogger(Firefox.class.getName()); private static final String historyQuery = "SELECT moz_historyvisits.id,url,title,visit_count,(visit_date/1000000) as visit_date,from_visit,(SELECT url FROM moz_places WHERE id=moz_historyvisits.from_visit) as ref FROM moz_places, moz_historyvisits WHERE moz_places.id = moz_historyvisits.place_id AND hidden = 0"; private static final String cookieQuery = "SELECT name,value,host,expiry,(lastAccessed/1000000) as lastAccessed,(creationTime/1000000) as creationTime FROM moz_cookies"; private static final String cookieQueryV3 = "SELECT name,value,host,expiry,(lastAccessed/1000000) as lastAccessed FROM moz_cookies"; @@ -59,21 +60,25 @@ class Firefox extends Extract { private static final String downloadQuery = "SELECT target, source,(startTime/1000000) as startTime, maxBytes FROM moz_downloads"; private static final String downloadQueryVersion24 = "SELECT url, content as target, (lastModified/1000000) as lastModified FROM moz_places, moz_annos WHERE moz_places.id = moz_annos.place_id AND moz_annos.anno_attribute_id = 3"; private final IngestServices services = IngestServices.getInstance(); + private Content dataSource; + private IngestJobContext context; Firefox() { moduleName = NbBundle.getMessage(Firefox.class, "Firefox.moduleName"); } @Override - public void process(Content dataSource, DataSourceIngestModuleStatusHelper controller) { + public void process(Content dataSource, IngestJobContext context) { + this.dataSource = dataSource; + this.context = context; dataFound = false; - this.getHistory(dataSource, controller); - this.getBookmark(dataSource, controller); - this.getDownload(dataSource, controller); - this.getCookie(dataSource, controller); + this.getHistory(); + this.getBookmark(); + this.getDownload(); + this.getCookie(); } - private void getHistory(Content dataSource, DataSourceIngestModuleStatusHelper controller) { + private void getHistory() { FileManager fileManager = currentCase.getServices().getFileManager(); List historyFiles; try { @@ -111,14 +116,14 @@ class Firefox extends Extract { continue; } File dbFile = new File(temps); - if (controller.isIngestJobCancelled()) { + if (context.isJobCancelled()) { dbFile.delete(); break; } List> tempList = this.dbConnect(temps, historyQuery); - logger.log(Level.INFO, "{0}- Now getting history from {1} with {2}artifacts identified.", new Object[]{moduleName, temps, tempList.size()}); + logger.log(Level.INFO, "{0} - Now getting history from {1} with {2} artifacts identified.", new Object[]{moduleName, temps, tempList.size()}); for (HashMap result : tempList) { - Collection bbattributes = new ArrayList(); + Collection bbattributes = new ArrayList<>(); bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL.getTypeID(), NbBundle.getMessage(this.getClass(), "Firefox.parentModuleName.noSpace"), @@ -155,14 +160,11 @@ class Firefox extends Extract { /** * Queries for bookmark files and adds artifacts - * - * @param dataSource - * @param controller */ - private void getBookmark(Content dataSource, DataSourceIngestModuleStatusHelper controller) { + private void getBookmark() { FileManager fileManager = currentCase.getServices().getFileManager(); - List bookmarkFiles = null; + List bookmarkFiles; try { bookmarkFiles = fileManager.findFiles(dataSource, "places.sqlite", "Firefox"); } catch (TskCoreException ex) { @@ -195,15 +197,15 @@ class Firefox extends Extract { continue; } File dbFile = new File(temps); - if (controller.isIngestJobCancelled()) { + if (context.isJobCancelled()) { dbFile.delete(); break; } List> tempList = this.dbConnect(temps, bookmarkQuery); - logger.log(Level.INFO, moduleName + "- Now getting bookmarks from " + temps + " with " + tempList.size() + "artifacts identified."); + logger.log(Level.INFO, "{0} - Now getting bookmarks from {1} with {2} artifacts identified.", new Object[]{moduleName, temps, tempList.size()}); for (HashMap result : tempList) { - Collection bbattributes = new ArrayList(); + Collection bbattributes = new ArrayList<>(); bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL.getTypeID(), NbBundle.getMessage(this.getClass(), "Firefox.parentModuleName.noSpace"), @@ -239,13 +241,10 @@ class Firefox extends Extract { /** * Queries for cookies file and adds artifacts - * - * @param dataSource - * @param controller */ - private void getCookie(Content dataSource, DataSourceIngestModuleStatusHelper controller) { + private void getCookie() { FileManager fileManager = currentCase.getServices().getFileManager(); - List cookiesFiles = null; + List cookiesFiles; try { cookiesFiles = fileManager.findFiles(dataSource, "cookies.sqlite", "Firefox"); } catch (TskCoreException ex) { @@ -278,12 +277,12 @@ class Firefox extends Extract { continue; } File dbFile = new File(temps); - if (controller.isIngestJobCancelled()) { + if (context.isJobCancelled()) { dbFile.delete(); break; } boolean checkColumn = Util.checkColumn("creationTime", "moz_cookies", temps); - String query = null; + String query; if (checkColumn) { query = cookieQuery; } else { @@ -291,10 +290,10 @@ class Firefox extends Extract { } List> tempList = this.dbConnect(temps, query); - logger.log(Level.INFO, moduleName + "- Now getting cookies from " + temps + " with " + tempList.size() + "artifacts identified."); + logger.log(Level.INFO, "{0} - Now getting cookies from {1} with {2} artifacts identified.", new Object[]{moduleName, temps, tempList.size()}); for (HashMap result : tempList) { - Collection bbattributes = new ArrayList(); + Collection bbattributes = new ArrayList<>(); bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL.getTypeID(), NbBundle.getMessage(this.getClass(), "Firefox.parentModuleName.noSpace"), @@ -339,27 +338,21 @@ class Firefox extends Extract { /** * Queries for downloads files and adds artifacts - * - * @param dataSource - * @param controller */ - private void getDownload(Content dataSource, DataSourceIngestModuleStatusHelper controller) { - getDownloadPreVersion24(dataSource, controller); - getDownloadVersion24(dataSource, controller); + private void getDownload() { + getDownloadPreVersion24(); + getDownloadVersion24(); } /** * Finds downloads artifacts from Firefox data from versions before 24.0. * * Downloads were stored in a separate downloads database. - * - * @param dataSource - * @param controller */ - private void getDownloadPreVersion24(Content dataSource, DataSourceIngestModuleStatusHelper controller) { + private void getDownloadPreVersion24() { FileManager fileManager = currentCase.getServices().getFileManager(); - List downloadsFiles = null; + List downloadsFiles; try { downloadsFiles = fileManager.findFiles(dataSource, "downloads.sqlite", "Firefox"); } catch (TskCoreException ex) { @@ -392,7 +385,7 @@ class Firefox extends Extract { continue; } File dbFile = new File(temps); - if (controller.isIngestJobCancelled()) { + if (context.isJobCancelled()) { dbFile.delete(); break; } @@ -464,13 +457,10 @@ class Firefox extends Extract { * Gets download artifacts from Firefox data from version 24. * * Downloads are stored in the places database. - * - * @param dataSource - * @param controller */ - private void getDownloadVersion24(Content dataSource, DataSourceIngestModuleStatusHelper controller) { + private void getDownloadVersion24() { FileManager fileManager = currentCase.getServices().getFileManager(); - List downloadsFiles = null; + List downloadsFiles; try { downloadsFiles = fileManager.findFiles(dataSource, "places.sqlite", "Firefox"); } catch (TskCoreException ex) { @@ -504,17 +494,17 @@ class Firefox extends Extract { continue; } File dbFile = new File(temps); - if (controller.isIngestJobCancelled()) { + if (context.isJobCancelled()) { dbFile.delete(); break; } List> tempList = this.dbConnect(temps, downloadQueryVersion24); - logger.log(Level.INFO, moduleName + "- Now getting downloads from " + temps + " with " + tempList.size() + "artifacts identified."); + logger.log(Level.INFO, "{0} - Now getting downloads from {1} with {2} artifacts identified.", new Object[]{moduleName, temps, tempList.size()}); for (HashMap result : tempList) { - Collection bbattributes = new ArrayList(); + Collection bbattributes = new ArrayList<>(); bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL.getTypeID(), NbBundle.getMessage(this.getClass(), diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RAImageIngestModule.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RAImageIngestModule.java index 3275252ae9..5c9ea7271b 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RAImageIngestModule.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RAImageIngestModule.java @@ -30,7 +30,7 @@ import org.openide.util.NbBundle; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.ingest.DataSourceIngestModule; -import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleStatusHelper; +import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProgress; import org.sleuthkit.autopsy.ingest.IngestServices; import org.sleuthkit.autopsy.ingest.IngestMessage; import org.sleuthkit.autopsy.ingest.IngestMessage.MessageType; @@ -45,21 +45,19 @@ import org.sleuthkit.autopsy.ingest.IngestJobContext; public final class RAImageIngestModule extends IngestModuleAdapter implements DataSourceIngestModule { private static final Logger logger = Logger.getLogger(RAImageIngestModule.class.getName()); - private static int messageId = 0; private final List extracters = new ArrayList<>(); private final List browserExtracters = new ArrayList<>(); private IngestServices services = IngestServices.getInstance(); + private IngestJobContext context; private StringBuilder subCompleted = new StringBuilder(); RAImageIngestModule() { } - synchronized int getNextMessageId() { - return ++messageId; - } - @Override public void startUp(IngestJobContext context) throws IngestModuleException { + this.context = context; + Extract registry = new ExtractRegistry(); Extract iexplore = new ExtractIE(); Extract recentDocuments = new RecentDocumentsByLnk(); @@ -84,22 +82,22 @@ public final class RAImageIngestModule extends IngestModuleAdapter implements Da } @Override - public ProcessResult process(Content dataSource, DataSourceIngestModuleStatusHelper controller) { - services.postMessage(IngestMessage.createMessage(getNextMessageId(), MessageType.INFO, RecentActivityExtracterModuleFactory.getModuleName(), "Started " + dataSource.getName())); + public ProcessResult process(Content dataSource, DataSourceIngestModuleProgress progressBar) { + services.postMessage(IngestMessage.createMessage(MessageType.INFO, RecentActivityExtracterModuleFactory.getModuleName(), "Started " + dataSource.getName())); - controller.switchToDeterminate(extracters.size()); - controller.progress(0); + progressBar.switchToDeterminate(extracters.size()); + progressBar.progress(0); ArrayList errors = new ArrayList<>(); for (int i = 0; i < extracters.size(); i++) { Extract extracter = extracters.get(i); - if (controller.isIngestJobCancelled()) { + if (context.isJobCancelled()) { logger.log(Level.INFO, "Recent Activity has been canceled, quitting before {0}", extracter.getName()); break; } try { - extracter.process(dataSource, controller); + extracter.process(dataSource, context); } catch (Exception ex) { logger.log(Level.SEVERE, "Exception occurred in " + extracter.getName(), ex); subCompleted.append(NbBundle.getMessage(this.getClass(), "RAImageIngestModule.process.errModFailed", @@ -107,7 +105,7 @@ public final class RAImageIngestModule extends IngestModuleAdapter implements Da errors.add( NbBundle.getMessage(this.getClass(), "RAImageIngestModule.process.errModErrs", RecentActivityExtracterModuleFactory.getModuleName())); } - controller.progress(i + 1); + progressBar.progress(i + 1); errors.addAll(extracter.getErrorMessages()); } @@ -134,7 +132,7 @@ public final class RAImageIngestModule extends IngestModuleAdapter implements Da errorMessage.append(NbBundle.getMessage(this.getClass(), "RAImageIngestModule.process.errMsg.noErrs")); errorMsgSubject = NbBundle.getMessage(this.getClass(), "RAImageIngestModule.process.errMsgSub.noErrs"); } - final IngestMessage msg = IngestMessage.createMessage(getNextMessageId(), msgLevel, RecentActivityExtracterModuleFactory.getModuleName(), + final IngestMessage msg = IngestMessage.createMessage(msgLevel, RecentActivityExtracterModuleFactory.getModuleName(), NbBundle.getMessage(this.getClass(), "RAImageIngestModule.process.ingestMsg.finished", dataSource.getName(), errorMsgSubject), @@ -152,7 +150,7 @@ public final class RAImageIngestModule extends IngestModuleAdapter implements Da historyMsg.append(""); } historyMsg.append(""); - final IngestMessage inboxMsg = IngestMessage.createMessage(getNextMessageId(), MessageType.INFO, RecentActivityExtracterModuleFactory.getModuleName(), + final IngestMessage inboxMsg = IngestMessage.createMessage(MessageType.INFO, RecentActivityExtracterModuleFactory.getModuleName(), NbBundle.getMessage(this.getClass(), "RAImageIngestModule.process.ingestMsg.results", dataSource.getName()), diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RecentDocumentsByLnk.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RecentDocumentsByLnk.java index 47827a7ce0..bfa52c250a 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RecentDocumentsByLnk.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RecentDocumentsByLnk.java @@ -32,7 +32,8 @@ import java.util.Collection; import org.sleuthkit.autopsy.coreutils.JLNK; import org.sleuthkit.autopsy.coreutils.JLnkParser; import org.sleuthkit.autopsy.coreutils.JLnkParserException; -import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleStatusHelper; +import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProgress; +import org.sleuthkit.autopsy.ingest.IngestJobContext; import org.sleuthkit.autopsy.ingest.IngestServices; import org.sleuthkit.autopsy.ingest.ModuleDataEvent; import org.sleuthkit.datamodel.BlackboardArtifact; @@ -48,14 +49,16 @@ import org.sleuthkit.datamodel.*; */ class RecentDocumentsByLnk extends Extract { private static final Logger logger = Logger.getLogger(RecentDocumentsByLnk.class.getName()); - private IngestServices services = IngestServices.getInstance(); + private IngestServices services = IngestServices.getInstance(); + private Content dataSource; + private IngestJobContext context; /** * Find the documents that Windows stores about recent documents and make artifacts. * @param dataSource * @param controller */ - private void getRecentDocuments(Content dataSource, DataSourceIngestModuleStatusHelper controller) { + private void getRecentDocuments() { org.sleuthkit.autopsy.casemodule.services.FileManager fileManager = currentCase.getServices().getFileManager(); List recentFiles; @@ -76,14 +79,14 @@ class RecentDocumentsByLnk extends Extract { dataFound = true; for (AbstractFile recentFile : recentFiles) { - if (controller.isIngestJobCancelled()) { + if (context.isJobCancelled()) { break; } if (recentFile.getSize() == 0) { continue; } - JLNK lnk = null; + JLNK lnk; JLnkParser lnkParser = new JLnkParser(new ReadContentInputStream(recentFile), (int) recentFile.getSize()); try { lnk = lnkParser.parse(); @@ -100,7 +103,7 @@ class RecentDocumentsByLnk extends Extract { continue; } - Collection bbattributes = new ArrayList(); + Collection bbattributes = new ArrayList<>(); String path = lnk.getBestPath(); bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PATH.getTypeID(), NbBundle.getMessage(this.getClass(), @@ -122,8 +125,10 @@ class RecentDocumentsByLnk extends Extract { } @Override - public void process(Content dataSource, DataSourceIngestModuleStatusHelper controller) { + public void process(Content dataSource, IngestJobContext context) { + this.dataSource = dataSource; + this.context = context; dataFound = false; - this.getRecentDocuments(dataSource, controller); + this.getRecentDocuments(); } } diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java index b57eb19a66..6bdd730345 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java @@ -31,11 +31,11 @@ import java.util.logging.Level; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; -import org.sleuthkit.autopsy.coreutils.Logger; import org.openide.util.NbBundle; +import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.PlatformUtil; import org.sleuthkit.autopsy.coreutils.XMLUtil; -import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleStatusHelper; +import org.sleuthkit.autopsy.ingest.IngestJobContext; import org.sleuthkit.autopsy.ingest.IngestModule.IngestModuleException; import org.sleuthkit.autopsy.ingest.IngestServices; import org.sleuthkit.autopsy.ingest.ModuleDataEvent; @@ -62,6 +62,7 @@ import org.xml.sax.SAXException; */ class SearchEngineURLQueryAnalyzer extends Extract { + private static final Logger logger = Logger.getLogger(SearchEngineURLQueryAnalyzer.class.getName()); private static final String XMLFILE = "SEUQAMappings.xml"; private static final String XSDFILE = "SearchEngineSchema.xsd"; private static String[] searchEngineNames; @@ -71,6 +72,8 @@ class SearchEngineURLQueryAnalyzer extends Extract { NbBundle.getMessage(SearchEngineURLQueryAnalyzer.class, "SearchEngineURLQueryAnalyzer.engineName.none"), NbBundle.getMessage(SearchEngineURLQueryAnalyzer.class, "SearchEngineURLQueryAnalyzer.domainSubStr.none"), new HashMap()); + private Content dataSource; + private IngestJobContext context; SearchEngineURLQueryAnalyzer() { } @@ -223,7 +226,7 @@ class SearchEngineURLQueryAnalyzer extends Extract { return basereturn; } - private void getURLs(Content dataSource, DataSourceIngestModuleStatusHelper controller) { + private void getURLs() { int totalQueries = 0; try { //from blackboard_artifacts @@ -255,7 +258,7 @@ class SearchEngineURLQueryAnalyzer extends Extract { Collection listAttributes = currentCase.getSleuthkitCase().getMatchingAttributes("Where `artifact_id` = " + artifact.getArtifactID()); getAttributes: for (BlackboardAttribute attribute : listAttributes) { - if (controller.isIngestJobCancelled()) { + if (context.isJobCancelled()) { break getAll; //User cancled the process. } if (attribute.getAttributeTypeID() == BlackboardAttribute.ATTRIBUTE_TYPE.TSK_URL.getTypeID()) { @@ -292,13 +295,13 @@ class SearchEngineURLQueryAnalyzer extends Extract { } catch (TskException e) { logger.log(Level.SEVERE, "Encountered error retrieving artifacts for search engine queries", e); } finally { - if (controller.isIngestJobCancelled()) { + if (context.isJobCancelled()) { logger.info("Operation terminated by user."); } IngestServices.getInstance().fireModuleDataEvent(new ModuleDataEvent( NbBundle.getMessage(this.getClass(), "SearchEngineURLQueryAnalyzer.parentModuleName.noSpace"), BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_SEARCH_QUERY)); - logger.info("Extracted " + totalQueries + " queries from the blackboard"); + logger.log(Level.INFO, "Extracted {0} queries from the blackboard", totalQueries); } } @@ -314,15 +317,17 @@ class SearchEngineURLQueryAnalyzer extends Extract { } @Override - public void process(Content dataSource, DataSourceIngestModuleStatusHelper controller) { - this.getURLs(dataSource, controller); + public void process(Content dataSource, IngestJobContext context) { + this.dataSource = dataSource; + this.context = context; + this.getURLs(); logger.log(Level.INFO, "Search Engine stats: \n{0}", getTotals()); } @Override void init() throws IngestModuleException { try { - PlatformUtil.extractResourceToUserConfigDir(SearchEngineURLQueryAnalyzer.class, XMLFILE); + PlatformUtil.extractResourceToUserConfigDir(SearchEngineURLQueryAnalyzer.class, XMLFILE, false); init2(); } catch (IOException e) { String message = "Unable to find " + XMLFILE; diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/USB_DATA.txt b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/USB_DATA.txt index 2ffa37f7b9..c8a83c862b 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/USB_DATA.txt +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/USB_DATA.txt @@ -9,8 +9,8 @@ # The latest version can be obtained from # http://www.linux-usb.org/usb.ids # -# Version: 2013.05.24 -# Date: 2013-05-24 20:34:03 +# Version: 2014.02.03 +# Date: 2014-02-03 20:34:03 # # Vendors, devices and interfaces. Please keep sorted. @@ -26,10 +26,12 @@ 0002 Ingram 0003 Club Mac 0004 Nebraska Furniture Mart +0011 Unknown manufacturer + 7788 Flash mass storage drive 0053 Planex 5301 GW-US54ZGL 802.11bg 0079 DragonRise Inc. - 0006 Generic USB Joystick + 0006 PC TWIN SHOCK Gamepad 0011 Gamepad 0105 Trust International B.V. 145f NW-3100 802.11b/g 54Mbps Wireless Network Adapter [zd1211] @@ -37,8 +39,10 @@ 0112 Card Reader 017c MLK 145f Trust Deskset +0200 TP-Link + 0201 MA180 UMTS Modem 0204 Chipsbank Microelectronics Co., Ltd - 6025 CBM2080 Flash drive controller + 6025 CBM2080 / CBM2090 Flash drive controller 6026 CBM1180 Flash drive controller 0218 Hangzhou Worlde 0301 MIDI Port @@ -105,9 +109,12 @@ 2107 AVR Dragon 2109 STK541 ZigBee Development Board 210d XPLAIN evaluation kit (CDC ACM) + 2110 AVR JTAGICE3 Debugger and Programmer 2122 XMEGA-A1 Explained evaluation kit 2310 EVK11xx evaluation board 2fe4 ATxmega32A4U DFU bootloader + 2ff0 atmega32u2 DFU bootloader + 2ffa at90usb162 DFU bootloader 2ffb at90usb AVR DFU bootloader 2ffd at89c5130/c5131 DFU bootloader 2fff at89c5132/c51snd1c DFU bootloader @@ -189,6 +196,8 @@ 0412 Printing Support 0417 LaserJet 1200 series 0423 HS-COMBO Cardreader + 042a LaserJet M1132 MFP + 0441 HP Prime Calculator 0504 DeskJet 885c 0505 ScanJet 2100c 0507 DVD+RW @@ -219,6 +228,7 @@ 0912 Printing Support 0917 LaserJet 3330 0924 Modular Smartcard Keyboard + 094a Optical Mouse [672662-001] 0a01 ScanJet 2400c 0a17 color LaserJet 3700 0b01 ScanJet 82x0C @@ -335,7 +345,7 @@ 2311 OfficeJet d series 2312 OfficeJet Pro L7700 2317 LaserJet 4350 - 231d 4 GB Flash Drive + 231d Broadcom 2070 Bluetooth Combo 2402 PhotoSmart 7700 series 2404 Deskjet F2280 series 2405 ScanJet 4070 PhotoSmart @@ -346,6 +356,7 @@ 2504 DeskJet F4200 series 2505 ScanJet 3770 2512 OfficeJet Pro L7300 + 2514 4-port hub 2517 LaserJet 2410 251d Gobi 2000 Wireless Modem 2524 LP3065 30" Monitor Hub @@ -421,6 +432,7 @@ 3b11 PSC 1300 series 3b17 LaserJet M1005 MFP 3c02 PhotoSmart 7350 + 3c05 Scanjet Professional 1000 Mobile Scanner 3c11 PSC 1358 3c17 EWS UPD 3d02 PhotoSmart 7350~ @@ -470,6 +482,7 @@ 5017 EWS UPD 5111 PhotoSmart 3200 series 5211 PhotoSmart 3300 series + 5307 v165w Stick 5311 OfficeJet 6300 5312 Officejet Pro 8500A 5411 OfficeJet 4300 @@ -689,11 +702,20 @@ 8028 Dev board JTAG (FT232H based) 8040 4 Port Hub 8070 7 Port Hub + 8140 Vehicle Explorer Interface + 8210 MGTimer - MGCC (Vic) Timing System 8370 7 Port Hub 8371 PS/2 Keyboard And Mouse 8372 FT8U100AX Serial Port 8a28 Rainforest Automation ZigBee Controller 8a98 TIAO Multi-Protocol Adapter + 8b28 Alpermann+Velte TCI70 + 8b29 Alpermann+Velte TC60 CLS + 8b2a Alpermann+Velte Rubidium Q1 + 8b2b Alpermann+Velte TCD + 8b2c Alpermann+Velte TCC70 + 9133 CallerID + 9135 Rotary Pub alarm 9e90 Marvell OpenRD Base/Client 9f80 Ewert Energy Systems CANdapter a6d0 Texas Instruments XDS100v2 JTAG / BeagleBone A3 @@ -726,6 +748,7 @@ c8bc Alpermann+Velte Rubidium S1 c8bd Alpermann+Velte Rubidium T1 c8be Alpermann+Velte Rubidium D1 + c8bf Alpermann+Velte TC60 RLV cc48 Tactrix OpenPort 1.3 Mitsubishi cc49 Tactrix OpenPort 1.3 Subaru cc4a Tactrix OpenPort 1.3 Universal @@ -994,7 +1017,10 @@ 05d4 EasyShare C180 Digital Camera 1001 EasyShare SV811 Digital Picture Frame 4000 InkJet Color Printer + 4021 Photo Printer 6800 4022 1400 Digital Photo Printer + 402e 605 Photo Printer + 4034 805 Photo Printer 4056 ESP 7200 Series AiO 4109 EasyShare Printer Dock Series 3 410d EasyShare G600 Printer Dock @@ -1006,6 +1032,7 @@ 6004 i60 6005 i80 040b Weltrend Semiconductor + 0a68 Func MS-3 gaming mouse [WT6573F MCU] 6510 Weltrend Bar Code Reader 6520 XBOX Xploder 6533 Speed-Link Competition Pro @@ -1234,6 +1261,7 @@ 4056 Live! Cam Video IM Pro 4057 Live! Cam Optia 4058 Live! Cam Optia AF + 405f WebCam Vista (VF0330) 4061 Live! Cam Notebook Pro [VF0400] 4063 Live! Cam Video IM Pro 4068 Live! Cam Notebook [VF0470] @@ -1450,7 +1478,7 @@ 4060 Ultra Fast Media Reader 4064 Ultra Fast Media Reader 7500 LAN7500 Ethernet 10/100/1000 Adapter - 9512 LAN9500 Ethernet 10/100 Adapter / SMSC9512/9514 Hub + 9512 SMC9512/9514 USB Hub a700 2 Port Hub ec00 SMSC9512/9514 Fast Ethernet Adapter 0425 Motorola Semiconductors HK, Ltd @@ -1645,6 +1673,7 @@ 010d 3500-4500 series 010f 6500 series 0142 X3650 (Printer, Scanner, Copier) + 01fa S310 series 4303 Xerox WorkCentre Pro 412 043e LG Electronics USA, Inc. 3001 AN-WF100 802.11abgn Wireless Adapter [Broadcom BCM4323] @@ -1652,6 +1681,8 @@ 4a4d Flatron 915FT Plus Monitor 7001 MF-PD100 Soul Digital MP3 Player 7013 MP3 Player + 70d7 Mouse Scanner LSM-150 [LG Smart Scan Mouse] + 70f5 External HDD 8484 LPC-U30 Webcam II 8585 LPC-UC35 Webcam 8888 Electronics VCS Camera II(LPC-U20) @@ -1706,6 +1737,7 @@ b10a T.16000M Joystick b203 360 Modena Pro Wheel b300 Firestorm Dual Power + b303 FireStorm Dual Analog 2 b304 Firestorm Dual Power b307 vibrating Upad b30b Wireless VibrationPad @@ -1738,10 +1770,11 @@ 625f TUSB6250 ATA Bridge 8042 Hub dbc0 Device Bay Controller - e001 GraphLink + e001 GraphLink [SilverLink] e003 TI-84 Plus Calculator e004 TI-89 Titanium Calculator e008 TI-84 Plus Silver Calculator + e012 TI-Nspire Calculator f430 MSP-FET430UIF JTAG Tool f432 eZ430 Development Tool ffff Bluetooth Device @@ -1762,6 +1795,7 @@ 0151 Super Flash 1GB / GXT 64MB Flash Drive 0162 SiS162 usb Wireless LAN Adapter 0163 802.11 Wireless LAN Adapter + 0817 SiS-184-ASUS-4352.17 touch panel 5401 Wireless Adapter RO80211GS-USB 0458 KYE Systems Corp. (Mouse Systems) 0001 Mouse @@ -1854,6 +1888,7 @@ 705c Genius iSlim 1300AF 7079 FaceCam 2025R 707f TVGo DVB-T03 [RTL2832] + 7088 WideCam 1050 0459 Adobe Systems, Inc. 045a SONICblue, Inc. 07da Supra Express 56K modem @@ -1888,7 +1923,7 @@ 0008 SideWinder Precision Pro 0009 IntelliMouse 000b Natural Keyboard Elite - 000e SideWinder® Freestyle Pro + 000e SideWinder® Freestyle Pro 0014 Digital Sound System 80 001a SideWinder Precision Racing Wheel 001b SideWinder Force Feedback 2 Joystick @@ -1984,10 +2019,12 @@ 029d Xbox360 HD-DVD Drive 029e Xbox360 HD-DVD Memory Unit 02a0 Xbox360 Big Button IR + 02a1 Xbox 360 Wireless Receiver for Windows 02a8 Xbox360 Wireless N Networking Adapter [Atheros AR7010+AR9280] 02ad Xbox NUI Audio 02ae Xbox NUI Camera 02b0 Xbox NUI Motor + 02b6 Xbox 360 / Bluetooth Wireless Headset 0400 Windows Powered Pocket PC 2002 0401 Windows Powered Pocket PC 2002 0402 Windows Powered Pocket PC 2002 @@ -2142,7 +2179,10 @@ 076d LifeCam HD-5000 0772 LifeCam Studio 0779 LifeCam HD-3000 + 0780 Comfort Curve Keyboard 3000 + 0797 Optical Mouse 200 930a ISOUSB.SYS Intel 82930 Isochronous IO Test Board + ffca Catalina fff8 Keyboard ffff Windows CE Mass Storage 0460 Ace Cad Enterprise Co., Ltd @@ -2215,7 +2255,7 @@ 0467 AT&T Paradyne 0468 Wieson Technologies Co., Ltd 046a Cherry GmbH - 0001 My3000 Keyboard + 0001 Keyboard 0003 My3000 Hub 0004 CyBoard Keyboard 0005 XX33 SmartCard Reader Keyboard @@ -2228,6 +2268,7 @@ 002a Wireless Mouse & Keyboard 002d SmartTerminal XX44 003e SmartTerminal ST-2xxx + 0041 G86 6240 Keyboard 0080 eHealth Terminal ST 1503 0081 eHealth Keyboard G87 1504 0106 R-300 Wireless Mouse Receiver @@ -2367,6 +2408,7 @@ 0a13 Z-5 Speakers 0a17 G330 Headset 0a1f G930 + 0a29 H600 [Wireless Headset] 0b02 C-UV35 [Bluetooth Mini-Receiver] (HID proxy mode) 8801 Video Camera b305 BT Mini-Receiver @@ -2430,20 +2472,25 @@ c061 RX1500 Laser Mouse c062 M-UAS144 [LS1 Laser Mouse] c063 DELL Laser Mouse + c064 M110 corded optical mouse (M-B0001) + c066 G9x Laser Mouse c068 G500 Laser Mouse c069 M500 Laser Mouse c06a USB Optical Mouse c06b G700 Wireless Gaming Mouse c06c Optical Mouse + c077 M105 Optical Mouse c101 UltraX Media Remote c110 Harmony 785/885 Remote c111 Harmony 525 Remote c112 Harmony 890 Remote c11f Harmony 900/1100 Remote c121 Harmony One Remote - c122 Harmony 700 Remote + c122 Harmony 650/700 Remote c124 Harmony 300 Remote c125 Harmony 200 Remote + c126 Harmony Link + c12b Harmony Touch Remote c201 WingMan Extreme Joystick with Throttle c202 WingMan Formula c207 WingMan Extreme Digital 3D @@ -2473,12 +2520,18 @@ c225 G11/G15 Keyboard / G keys c226 G15 Refresh Keyboard c227 G15 Refresh Keyboard + c228 G19 Gaming Keyboard + c229 G19 Gaming Keyboard Macro Interface c22a Gaming Keyboard G110 c22b Gaming Keyboard G110 G-keys c22d G510 Gaming Keyboard c22e G510 Gaming Keyboard onboard audio c245 G400 Optical Mouse c246 Gaming Mouse G300 + c248 G105 Gaming Keyboard + c24a G600 Gaming Mouse + c24d G710 Gaming Keyboard + c24e G500s Laser Gaming Mouse c281 WingMan Force c283 WingMan Force 3D c285 WingMan Strike Force 3D @@ -2550,12 +2603,14 @@ c526 Nano Receiver c529 Logitech Keyboard + Mice c52b Unifying Receiver + c52e MK260 Wireless Combo Receiver c52f Unifying Receiver c532 Unifying Receiver c623 3Dconnexion Space Traveller 3D Mouse c625 3Dconnexion Space Pilot 3D Mouse c626 3Dconnexion Space Navigator 3D Mouse c627 3Dconnexion Space Explorer 3D Mouse + c629 3Dconnexion SpacePilot Pro 3D Mouse c702 Cordless Presenter c703 Elite Keyboard Y-RP20 + Mouse MX900 (Bluetooth) c704 diNovo Wireless Desktop @@ -2694,6 +2749,7 @@ 20b6 GoGear Vibe 20d0 SPZ2000 Webcam [PixArt PAC7332] 20e3 GoGear Raga + 20e4 GoGear ViBE 8GB 262c SPC230NC Webcam 485d Senselock SenseIV v2.x df55 LPCXpresso LPC-Link @@ -2827,15 +2883,20 @@ 4254 BUA-100 Bluetooth Adapter ac01 Savi 7xx ad01 GameCom 777 5.1 Headset + c00e Blackwire C310 headset 0480 Toshiba America Info. Systems, Inc. 0001 InTouch Module 0004 InTouch Module 0011 InTouch Module 0014 InTouch Module + a006 External Disk 1.5TB a007 External Disk USB 3.0 + a009 Stor.E Basics + d010 External Disk 3TB 0481 Zenith Data Systems 0482 Kyocera Corp. 000e FS-1020D Printer + 000f FS-1920 Mono Printer 0100 Finecam S3x 0101 Finecam S4 0103 Finecam S5 @@ -2850,7 +2911,7 @@ 0138 Unicorn II (ST70138B + MTC-20174TQ chipset) 1307 Cytronix 6in1 Card Reader 163d Cool Icam Digi-MP3 - 2015 TouchChip® Fingerprint Reader + 2015 TouchChip® Fingerprint Reader 2016 Fingerprint Reader 2017 Biometric Smart Card Reader 2018 BioSimKey @@ -2863,6 +2924,7 @@ 5000 ST Micro/Ergenic ERG BT-002 Bluetooth Adapter 5001 ST Micro Bluetooth Device 5710 Joystick in FS Mode + 5720 STM microSD Flash Device 5721 Hantek DDS-3X25 Arbitrary Waveform Generator 5740 STM32F407 7270 ST Micro Serial Bridge @@ -2895,6 +2957,8 @@ 9006 IT9135 BDA Afatech DVB-T HDTV Dongle 9009 Zolid HD DVD Maker 9135 Zolid Mini DVB-T Stick + 9503 ITE it9503 feature-limited DVB-T transmission chip [ccHDtv] + 9507 ITE it9507 full featured DVB-T transmission chip [ccHDtv] 048f Eicon Tech. 0490 United Microelectronics Corp. 0491 Capetronic @@ -3253,12 +3317,14 @@ 171b MP960 1721 MP210 ser 1723 MP470 ser + 1724 PIXMA MP520 series 1725 MP610 ser 1726 MP970 ser 1727 MX300 ser 1728 MX310 ser 1729 MX700 ser 172b MP140 ser + 173b PIXMA MP270 All-In-One Printer 173e MP560 173f Pixma MP640 Multifunction device 1748 Pixma MG5150 @@ -3500,34 +3566,65 @@ 3100 PowerShot TX1 310b SELPHY CP600 310e Digital IXUS 50 (PTP mode) + 310f PowerShot A420 3110 EOS Digital Rebel XTi + 3115 PowerShot SD900 / Digital IXUS 900 Ti / IXY DIGITAL 1000 3116 Digital IXUS 750 / PowerShot SD550 (PTP mode) 3117 PowerShot A700 3119 PowerShot SD700 IS / Digital IXUS 800 IS / IXY Digital 800 IS + 311a PowerShot S3 IS 311b PowerShot A540 + 311c PowerShot SD600 DIGITAL ELPH / DIGITAL IXUS 60 / IXY DIGITAL 70 + 3125 PowerShot G7 + 3126 PowerShot A530 3127 SELPHY CP710 3128 SELPHY CP510 312d Elura 100 + 3136 PowerShot SD800 IS / Digital IXUS 850 IS / IXY DIGITAL 900 IS + 3137 PowerShot SD40 / Digital IXUS i7 IXY / DIGITAL L4 3138 PowerShot A710 IS + 3139 PowerShot A640 + 313a PowerShot A630 3141 SELPHY ES1 3142 SELPHY CP730 3143 SELPHY CP720 3145 EOS 450D + 3146 EOS 40D 3147 EOS 1Ds Mark III - 314f Powershot SD1000 + 3148 PowerShot S5 IS + 3149 PowerShot A460 + 314b PowerShot SD850 IS DIGITAL ELPH / Digital IXUS 950 IS / IXY DIGITAL 810 IS + 314c PowerShot A570 IS + 314d PowerShot A560 + 314e PowerShot SD750 DIGITAL ELPH / DIGITAL IXUS 75 / IXY DIGITAL 90 + 314f PowerShot SD1000 DIGITAL ELPH / DIGITAL IXUS 70 / IXY DIGITAL 10 + 3150 PowerShot A550 3155 PowerShot A450 315a PowerShot G9 + 315b PowerShot A650 IS 315d PowerShot A720 + 315e PowerShot SX100 IS + 315f PowerShot SD950 IS DIGITAL ELPH / DIGITAL IXUS 960 IS / IXY DIGITAL 2000 IS 3160 Digital IXUS 860 IS 3170 SELPHY CP750 3171 SELPHY CP740 + 3173 PowerShot SD890 IS DIGITAL ELPH / Digital IXUS 970 IS / IXY DIGITAL 820 IS + 3174 PowerShot SD790 IS DIGITAL ELPH / Digital IXUS 90 IS / IXY DIGITAL 95 IS 3175 IXY Digital 25 IS 3176 PowerShot A590 + 3177 PowerShot A580 317a PC1267 [Powershot A470] 3184 Digital IXUS 80 IS (PTP mode) 3185 SELPHY ES2 3186 SELPHY ES20 + 318d PowerShot SX100 IS + 318e PowerShot A1000 IS + 318f PowerShot G10 + 3191 PowerShot A2000 IS 3192 PowerShot SX110 IS + 3193 PowerShot SD990 IS DIGITAL ELPH / Digital IXUS 980 IS / IXY DIGITAL 3000 IS + 3195 PowerShot SX1 IS + 3196 PowerShot SD880 IS DIGITAL ELPH / Digital IXUS 870 IS / IXY DIGITAL 920 IS 319a EOS 7D 31aa SELPHY CP770 31ab SELPHY CP760 @@ -3536,12 +3633,27 @@ 31b0 SELPHY ES30 31b1 SELPHY CP530 31bc PowerShot D10 + 31bd PowerShot SD960 IS DIGITAL ELPH / Digital IXUS 110 IS / IXY DIGITAL 510 IS + 31be PowerShot A2100 IS 31bf PowerShot A480 31c0 PowerShot SX200 IS + 31c1 PowerShot SD970 IS DIGITAL ELPH / Digital IXUS 990 IS / IXY DIGITAL 830 IS + 31c2 PowerShot SD780 IS DIGITAL ELPH / Digital IXUS 100 IS / IXY DIGITAL 210 IS + 31c3 PowerShot A1100 IS + 31c4 PowerShot SD1200 IS DIGITAL ELPH / Digital IXUS 95 IS / IXY DIGITAL 110 IS + 31cf EOS Rebel T1i / EOS 500D / EOS Kiss X3 31dd SELPHY CP780 + 31df PowerShot G11 + 31e0 PowerShot SX120 IS + 31e1 PowerShot S90 + 31e4 PowerShot SX20 IS 31e5 Digital IXUS 200 IS + 31e6 PowerShot SD940 IS DIGITAL ELPH / Digital IXUS 120 IS / IXY DIGITAL 220 IS + 31e7 SELPHY CP790 + 31ea EOS Rebel T2i / EOS 550D / EOS Kiss X4 31ee SELPHY ES40 31ef PowerShot A495 + 31f0 PowerShot A490 31f1 PowerShot A3100 IS / PowerShot A3150 IS 31f2 PowerShot A3000 IS 31f3 PowerShot Digital ELPH SD1400 IS @@ -3552,6 +3664,7 @@ 31f8 Powershot SD4500 IS / IXUS 1000 HS / IXY 50S 31ff Digital IXUS 55 3209 Vixia HF S21 A + 320f PowerShot G12 3210 Powershot SX30 IS 3211 PowerShot SX130 IS 3212 Powershot S95 @@ -3559,11 +3672,14 @@ 3218 EOS 600D / Rebel T3i (ptp) 3223 PowerShot A3300 IS 3224 PowerShot A3200 IS + 3225 PowerShot ELPH 500 HS / IXUS 310 HS 3226 PowerShow A800 + 3227 PowerShot ELPH 100 HS / IXUS 115 HS 3228 PowerShot SX230 HS 3229 PowerShot ELPH 300 HS / IXUS 220 HS 322a PowerShot A2200 322b Powershot A1200 + 322c PowerShot SX220 HS 3233 PowerShot G1 X 3234 PowerShot SX150 IS 3236 PowerShot S100 @@ -3574,6 +3690,7 @@ 323f PowerShot A810 3240 PowerShot ELPH 320 HS / IXUS 240 HS 3241 PowerShot ELPH 110 HS / IXUS 125 HS + 3242 PowerShot D20 3243 PowerShot A4000 IS 3244 PowerShot SX260 HS 3245 PowerShot SX240 HS @@ -3588,6 +3705,16 @@ 325a PowerShot SX160 IS 325b PowerShot S110 325c PowerShot SX500 IS + 325f PowerShot SX280 HS + 3260 PowerShot SX270 HS + 3262 PowerShot A2600 + 3264 PowerShot A1400 + 3265 Powershot ELPH 130 IS / IXUS 140 + 3268 PowerShot ELPH 330 HS / IXUS 255 HS + 3271 PowerShot A2500 + 3276 PowerShot SX170 IS + 3277 PowerShot SX510 HS + 327d Powershot ELPH 115 IS / IXUS 132 04aa DaeWoo Telecom, Ltd 04ab Chromatic Research 04ac Micro Audiometrics Corp. @@ -3658,8 +3785,11 @@ 0413 D40 (mass storage mode) 041e D60 digital camera (mass storage mode) 0422 D700 (ptp) + 0423 D5000 0424 D3000 0425 D300S + 0428 D7000 + 0429 D5100 042a D800 (ptp) 0f03 PD-10 Wireless Printer Adapter 4000 Coolscan LS 40 ED @@ -3688,6 +3818,7 @@ 4427 Portable CD ROM 4482 Serial Converter 4485 Serial Converter + 4524 40 Character Vacuum Fluorescent Display 4525 Double sided CRT 4535 4610 Suremark Printer 4550 NVRAM (128 KB) @@ -3700,6 +3831,7 @@ 0001 Mouse 0002 CY7C63x0x Thermometer 0033 Mouse + 0060 Wireless optical mouse 0100 Cino FuzzyScan F760-B 0101 Keyboard/Hub 0102 Keyboard with APM @@ -4018,6 +4150,7 @@ 1041 fi-4120c Scanner 1042 fi-4220c Scanner 105b AH-F401U Air H device + 1084 PalmSecure Sensor V2 1096 fi-5110EOX 1097 fi-5110C 10ae fi-4120C2 @@ -4027,6 +4160,7 @@ 10e7 fi-5900C 10fe S500 1150 fi-6230 + 201d SATA 3.0 6Gbit/s Adaptor [GROOVY] 04c6 Toshiba America Electronic Components 04c7 Micro Macro Technologies 04c8 Konica Corp. @@ -4040,7 +4174,9 @@ 072c Revio KD20M 072d Revio KD410Z 04ca Lite-On Technology Corp. + 004f SK-9020 keyboard 1766 HID Monitor Controls + 2004 Bluetooth 4.0 [Broadcom BCM20702A0] 9304 Hub f01c TT1280DA DVB-T TV Tuner 04cb Fuji Photo Film Co., Ltd @@ -4186,6 +4322,7 @@ 8001 ICD2 in-circuit debugger 8101 PIC24F Starter Kit 8107 Microstick II + 9004 Microchip REAL ICE 900a PICkit3 c001 PicoLCD 20x4 f8da Hughski Ltd. ColorHug @@ -4204,10 +4341,13 @@ 1400 PS/2 keyboard + mouse controller 1503 Shortboard Lefty 1603 Keyboard + 1702 Keyboard LKS02 2013 Keyboard [Das Keyboard] 2221 Keyboard 2323 Keyboard + 2519 Shenzhen LogoTech 2.4GHz receiver 2832 1channel Telephone line recorder + 2834 HT82A834R Audio MCU a055 Keyboard 04da Panasonic (Matsushita) 0901 LS-120 Camera @@ -4300,6 +4440,7 @@ 04e1 Iiyama North America, Inc. 0201 Monitor Hub 04e2 Exar Corp. + 1410 XR21V1410 USB-UART IC 04e3 Zilog, Inc. 04e4 ACC Microelectronics 04e5 Promise Technology @@ -4358,24 +4499,25 @@ 5151 SCR338 Keyboard Smart Card Reader 5292 SCL011 RFID reader 5410 SCR35xx Smart Card Reader + 5591 SCL3711-NFC&RW e000 SCRx31 Reader e001 SCR331 SmartCard Reader e003 SPR532 PinPad SmartCard Reader 04e7 Elo TouchSystems 0001 TouchScreen 0002 Touchmonitor Interface 2600 Rev 2 - 0004 4000U CarrollTouch® Touchmonitor Interface - 0007 2500U IntelliTouch® Touchmonitor Interface - 0008 3000U AccuTouch® Touchmonitor Interface - 0009 4000U CarrollTouch® Touchmonitor Interface + 0004 4000U CarrollTouch® Touchmonitor Interface + 0007 2500U IntelliTouch® Touchmonitor Interface + 0008 3000U AccuTouch® Touchmonitor Interface + 0009 4000U CarrollTouch® Touchmonitor Interface 0020 Touchscreen Interface (2700) 0021 Touchmonitor Interface - 0030 4500U CarrollTouch® Touchmonitor Interface + 0030 4500U CarrollTouch® Touchmonitor Interface 0032 Touchmonitor Interface 0033 Touchmonitor Interface 0041 5010 Surface Capacitive Touchmonitor Interface 0042 Touchmonitor Interface - 0050 2216 AccuTouch® Touchmonitor Interface + 0050 2216 AccuTouch® Touchmonitor Interface 0071 Touchmonitor Interface 0072 Touchmonitor Interface 0081 Touchmonitor Interface @@ -4385,7 +4527,7 @@ 0100 Kingston Flash Drive (128MB) 0110 Connect3D Flash Drive 0111 Connect3D Flash Drive - 0300 E2530 Phone (Mass storage mode) + 0300 E2530 / GT-C3350 Phones (Mass storage mode) 1003 MP3 Player and Recorder 1006 SDC-200Z 130c NX100 @@ -4522,8 +4664,10 @@ 5f04 NEXiO Sync 5f05 STORY Station 1TB 6032 G2 Portable hard drive + 6034 G2 Portable hard drive 60b3 M2 Portable Hard Drive 60c4 M2 Portable Hard Drive USB 3.0 + 61b6 M3 Portable Hard Drive 1TB 6601 Mobile Phone 6602 Galaxy 6603 Galaxy @@ -4563,7 +4707,7 @@ 685b GT-I9100 Phone [Galaxy S II] (mass storage mode) 685c GT-I9250 Phone [Galaxy Nexus] 685d GT-I9100 Phone [Galaxy S II] (Download mode) - 685e GT-I9100 Phone [Galaxy S II] (USB Debugging mode) + 685e GT-I9100 / GT-C3350 Phones (USB Debugging mode) 6860 GT-I9100 Phone [Galaxy S II], GT-I9300 Phone [Galaxy S III], GT-P7500 [Galaxy Tab 10.1] 6865 GT-I9300 Phone [Galaxy S III] (PTP mode) 6866 GT-I9300 Phone [Galaxy S III] (debugging mode) @@ -4620,6 +4764,7 @@ 0760 Acer KU-0760 Keyboard 0841 HP Multimedia Keyboard 0860 2.4G Multimedia Wireless Kit + 1121 Periboard 717 Mini Wireless Keyboard a001 E-Video DC-100 Camera a120 ORITE CCD Webcam(PC370R) a121 ORITE CCD Webcam(PC370R) @@ -4656,6 +4801,7 @@ b057 integrated USB webcam b059 CKF7037 HP webcam b071 2.0M UVC Webcam / CNF7129 + b083 CKF7063 Webcam (HP) b091 Webcam b104 CNF7069 Webcam b107 CNF7070 Webcam @@ -4667,6 +4813,7 @@ b1b9 Asus Integrated Webcam b1cf Lenovo Integrated Camera b1d6 CNF9055 Toshiba Webcam + b1e4 Toshiba Integrated Webcam b213 Fujitsu Integrated Camera b217 Lenovo Integrated Camera (0.3MP) b221 integrated camera @@ -4678,7 +4825,9 @@ b2b9 Lenovo Integrated Camera UVC b2ea Integrated Camera [ThinkPad] b330 Asus 720p CMOS webcam + b354 UVC 1.00 device HD UVC WebCam 04f3 Elan Microelectronics Corp. + 000a Touchscreen 0103 ActiveJet K-2024 Multimedia Keyboard 01a4 Wireless Keyboard 0210 Optical Mouse @@ -4894,6 +5043,7 @@ 201a PT-18R P-touch label printer 201b QL-650TD P-Touch Label Printer 2027 QL-560 P-Touch Label Printer + 202b PT-7600 P-Touch Label Printer 2100 Card Reader Writer 04fa Dallas Semiconductor 2490 DS1490F 2-in-1 Fob, 1-Wire adapter @@ -4945,6 +5095,7 @@ 16e3 n30 Handheld Sync 3202 Liquid 3203 Liquid (Debug mode) + 3230 BeTouch E120 3317 Liquid 3325 Iconia tablet A500 3341 Iconia tablet A500 @@ -5015,6 +5166,7 @@ 0304 FSU304 USB 2.0 - 4 Ports Hub 0307 USB 2.0 - 7 ports Hub [FSU307] 0409 F5U409 Serial + 0416 Staples 12416 7 port desktop hub 0551 F6C550-AVR UPS 0706 2-N-1 7-Port Hub (Lower half) 0802 Nostromo n40 Gamepad @@ -5026,6 +5178,8 @@ 1004 F9L1004 802.11n Surf N300 XR Wireless Adapter [Realtek RTL8192CU] 1102 F7D1102 N150/Surf Micro Wireless Adapter v1000 [Realtek RTL8188CUS] 1103 F9L1103 N750 DB 802.11abgn 2x3:3 [Ralink RT3573] + 1106 F9L1106v1 802.11a/b/g/n/ac Wireless Adapter [Broadcom BCM43526] + 1109 F9L1109v1 802.11a/b/g/n/ac Wireless Adapter [Realtek RTL8812AU] 11f2 ISY Wireless Micro Adapter IWL 2000 [RTL8188CUS] 1202 F5U120-PC Parallel Printer Port 1203 F5U120-PC Serial Port @@ -5117,7 +5271,7 @@ a4a2 Linux-USB Ethernet/RNDIS Gadget a4a3 Linux-USB user-mode isochronous source/sink a4a4 Linux-USB user-mode bulk source/sink - a4a5 Linux-USB File Storage Gadget + a4a5 Pocketbook Pro 903 a4a6 Linux-USB Serial Gadget a4a7 Linux-USB Serial Gadget (CDC ACM mode) a4a8 Linux-USB Printer Gadget @@ -5180,6 +5334,7 @@ 0539 Shyh Shiun Terminals Co., Ltd 053a PrehKeyTec GmbH 0b00 Hub + 0b01 Preh MCI 3100 053b Global Village Communication 053c Institut of Microelectronic & Mechatronic Systems 053d Silicon Architect @@ -5247,6 +5402,7 @@ 2727 Xircom PGUNET USB-USB Bridge 2750 EZ-Link (EZLNKUSB.SYS) 2810 Cypress ATAPI Bridge + 4d90 AmScope MD1900 camera 7777 Bluetooth Device 9999 AN2131 uninitialized (?) 0548 Tyan Computer Corp. @@ -5360,6 +5516,7 @@ 01d0 DVD+RW External Drive DRU-700A 01d5 IC RECORDER 01de VRD-VC10 [Video Capture] + 01e8 UP-DR150 Photo Printer 01e9 Net MD 01ea Hi-MD WALKMAN 01ee IC RECORDER @@ -5427,6 +5584,7 @@ 0440 DSC-H55 0485 MHS-PM5 HD camcorder 04cb WALKMAN NWZ-E354 + 06bb WALKMAN NWZ-F805 1000 Wireless Buzz! Receiver 054d Try Corp. 054e Proside Corp. @@ -5472,6 +5630,7 @@ 4000 DSB-650 10Mbps Ethernet [klsi] 7000 Hub 7820 UC-2322 2xSerial Ports [mos7820] + 8021 CS1764A [CubiQ DVI KVMP Switch] 0558 Truevision, Inc. 1009 GW Instek GDS-1000 Oscilloscope 100a GW Instek GDS-1000A Oscilloscope @@ -5654,6 +5813,7 @@ 00d6 Bamboo Pen & Touch (CTH-460) 00db Bamboo Fun (CTH-661SE-NL) 00dd Bamboo Pen (CTL-470) + 00de CTH-470 [Bamboo Fun Pen & Touch] 00f6 Cintiq 24HD touch (DTH-2400) touchscreen 00f8 Cintiq 24HD touch (DTH-2400) tablet 0400 PenPartner 4x5 @@ -6248,6 +6408,7 @@ 0323 LaCie d2 Drive USB2 0421 Big Disk G465 0641 Mobile Hard Drive + 100c Rugged Triple Interface Mobile Hard Drive 1010 Desktop Hard Drive 1019 Desktop Hard Drive 1021 Little Disk @@ -6269,6 +6430,8 @@ 9722 Keyboard 9731 MCK-600W/MCK-800USB Keyboard 9783 Wireless Keypad + 9837 Targus Number Keypad + 9862 Targus Number Keypad (Composite Device) 9881 IR receiver [VRC-1100 Vista MCE Remote Control] 05a5 Sampo Technology Corp. 05a6 Cisco Systems, Inc. @@ -6385,6 +6548,7 @@ 1101 Speakers 1105 Audio in LED Cinema Display 1107 Thunderbolt Display Audio + 1112 FaceTime HD Camera (Display) 1201 3G iPod 1202 iPod 2G 1203 iPod 4.Gen Grayscale 40G @@ -6454,6 +6618,7 @@ 8240 Built-in IR Receiver 8241 Built-in IR Receiver 8242 Built-in IR Receiver + 8281 Bluetooth Host Controller 8286 Bluetooth Host Controller 8300 Built-in iSight (no firmware loaded) 8403 Internal Memory Card Reader @@ -6910,6 +7075,7 @@ 0723 GL827L SD/MMC/MS Flash Card Reader 0726 SD Card Reader 0727 microSD Reader/Writer + 0731 GL3310 SATA 3Gb/s Bridge Controller 0736 microSD Reader/Writer 0760 USB 2.0 Card Reader/Writer 0761 Genesys Mass Storage Device @@ -6961,7 +7127,9 @@ 05f7 RFC Distribution(s) PTE, Ltd 05f9 PSC Scanning, Inc. 1104 Magellan 2200VS + 2202 Point of Sale Handheld Scanner 2206 Datalogic Gryphon GFS4170 + 2601 Datalogin Magellan 1000i Barcode Scanner 2602 Datalogic Magellan 1100i Barcode Scanner 5204 Datalogic Gryphon GFS4170 (config mode) 05fa Siemens Telecommunications Systems, Ltd @@ -7065,6 +7233,9 @@ 0622 Iotech, Inc. 0623 Littelfuse, Inc. 0624 Avocent Corp. + 0248 Virtual Hub + 0249 Virtual Keyboard/Mouse + 0251 Virtual Mass Storage 0294 Dell 03R874 KVM dongle 0402 Cisco Virtual Keyboard and Mouse 0403 Cisco Virtual Mass Storage @@ -7124,6 +7295,7 @@ 1000 CD-ROM Drive 800d TASCAM Portastudio DP-01FX 800e TASCAM US-122L + 801d DR-100 8021 TASCAM US-122mkII d001 CD-R/RW Unit d002 CD-R/RW Unit @@ -7156,6 +7328,7 @@ a219 1.3M WebCam (notebook emachines E730, Acer sub-brand) c107 HP webcam [dv6-1190en] d101 Acer CrystalEye Webcam + d217 HP TrueVision HD e201 Lenovo Integrated Webcam e203 Lenovo Integrated Webcam e258 HP TrueVision HD Integrated Webcam @@ -7183,7 +7356,7 @@ 0658 Sigma Designs, Inc. 0659 Aethra 065a Optoelectronics Co., Ltd - 0001 Barcode scanner / NLV-1001 (keyboard mode) + 0001 Opticon OPR-2001 / NLV-1001 (keyboard mode) 0009 NLV-1001 (serial mode) / OPN-2001 [Opticon] 065b Tracewell Systems 065e Silicon Graphics @@ -7377,6 +7550,7 @@ 2517 Flash Disk Mass Storage Device 2528 Storage device (8gB thumb drive) 25a1 PL25A1 Host-Host Bridge + 2773 PL2773 SATAII bridge controller 3400 Hi-Speed Flash Disk with TruePrint AES3400 3500 Hi-Speed Flash Disk with TruePrint AES3500 3507 PL3507 ATAPI6 Bridge @@ -7641,6 +7815,11 @@ 06bc Oki Data Corp. 000b Okipage 14ex Printer 0027 Okipage 14e + 00f7 OKI B4600 Mono Printer + 015e OKIPOS 411/412 POS Printer + 01c9 OKI B430 Mono Printer + 020b OKI ES4140 Mono Printer + 02bb OKI PT390 POS Printer 0a91 B2500MFP (printer+scanner) 3801 B6100 Laser Printer 06bd AGFA-Gevaert NV @@ -7761,6 +7940,7 @@ 0622 LapLink Gold USB-USB Bridge [net1080] 06d1 Daewoo Electronics Co., Ltd 06d3 Mitsubishi Electric Corp. + 0284 FX-USB-AW/-BD RS482 Converters 0380 CP8000D Port 0381 CP770D Port 0385 CP900D Port @@ -7800,10 +7980,12 @@ 0014 Prolink Winscan Pro 2448U 06de Heisei Electronics Co., Ltd 06e0 Multi-Tech Systems, Inc. + 0319 MT9234ZBA-USB MultiModem ZBA f101 MT5634ZBA-USB MultiModemUSB (old firmware) f103 MT5634MU MultiMobileUSB f104 MT5634ZBA-USB MultiModemUSB (new firmware) f107 MT5634ZBA-USB-V92 MultiModemUSB + f120 MT9234ZBA-USB-CDC-ACM-XR MultiModem ZBA CDC-ACM-XR 06e1 ADS Technologies, Inc. 0008 UBS-10BT Ethernet [klsi] 0009 UBS-10BT Ethernet @@ -8405,15 +8587,21 @@ 8911 ScanHex SX-35c 0798 Optelec 0001 Braille Voyager + 0640 BC640 + 0680 BC680 0799 Altera 7651 Programming Unit 079b Sagem + 0024 MSO300/MSO301 Fingerprint Sensor + 0026 MSO350/MSO351 Fingerprint Sensor & SmartCard Reader 0027 USB-Serial Controller 002f Mobile 0030 Mobile Communication Device 0042 Mobile + 0047 CBM/MSO1300 Fingerprint Sensor 004a XG-760A 802.11bg 004b Wi-Fi 11g adapter + 0052 MSO1350 Fingerprint Sensor & SmartCard Reader 0056 Agfa AP1100 Photo Printer 005d Mobile Mass Storage 0062 XG-76NA 802.11bg @@ -8538,12 +8726,17 @@ 0114 C-350Z Camera 0118 Mju Mini Digital/Mju Digital 500 Camera / Stylus 850 SW 0184 P-S100 port + 0202 Foot Switch RS-26 0203 Digital Voice Recorder DW-90 0206 Digital Voice Recorder DS-330 0207 Digital Voice Recorder & Camera W-10 0209 Digital Voice Recorder DM-20 + 020b Digital Voice Recorder DS-4000 020d Digital Voice Recorder VN-240PC + 0211 Digital Voice Recorder DS-2300 + 0218 Foot Switch RS-28 0244 Digital Voice Recorder VN-8500PC + 024f Digital Voice Recorder DS-7000 0280 m:robe 100 07b5 Mega World International, Ltd 0017 Joystick @@ -8575,6 +8768,7 @@ 420a UF200 Ethernet 5301 GW-US54ZGL 802.11bg 6001 802.11bg + 8188 AboCom Systems Inc [WN2001 Prolink Wireless-N Nano Adapter] a001 WUG2200 802.11g Wireless Adapter [Envara WiND512] abc1 DU-E10 Ethernet [pegasus] b000 BWU613 @@ -9044,6 +9238,7 @@ 1002 Ethernet 1020 FA101 Fast Ethernet USB 1.1 1040 FA120 Fast Ethernet USB 2.0 [Asix AX88172 / AX8817x] + 1100 Managed Switch M4100 series, M5300 series, M7100 series 4110 MA111(v1) 802.11b Wireless [Intersil Prism 3.0] 4200 WG121(v1) 54 Mbps Wireless [Intersil ISL3886] 4210 WG121(v2) 54 Mbps Wireless [Intersil ISL3886] @@ -9335,7 +9530,7 @@ 0301 RNIS 08e4 Pioneer Corp. 08e5 Litronic -08e6 Gemplus +08e6 Gemalto (was Gemplus) 0001 GemPC-Touch 430 0430 GemPC430 SmartCard Reader 0432 GemPC432 SmartCard Reader @@ -9349,6 +9544,7 @@ 34ec Compact Smart Card Reader Writer 4433 GemPC433-Swap 5501 GemProx-PU Contactless Smart Card Reader + 5503 Prox-DU Contactless Interface ace0 UA HYBRID TOKEN 08e7 Pan-International Wire & Cable 08e8 Integrated Memory Logic @@ -9502,8 +9698,8 @@ 0001 Hard Drive Adapter (TPP) 0002 SigmaDrive Adapter (TPP) 0906 Faraday Technology Corp. -0908 ShenZhen SANZHAI Technology Co.,Ltd - 2701 Spy Pen VGA +0908 Siemens AG + 2701 ShenZhen SANZHAI Technology Co.,Ltd Spy Pen VGA 0909 Audio-Technica Corp. 090a Trumpion Microelectronics, Inc. 1001 T33520 Flash Card Controller @@ -9582,9 +9778,9 @@ 2295 Colorado 300 22b6 eTrex Vista HCx (Mass Storage mode) 231b Oregon 400t - 2353 Nüvi 205T + 2353 Nüvi 205T 2380 Oregon series - 23cc nüvi 1350 + 23cc nüvi 1350 2459 GPSmap 62/78 series 2519 eTrex 30 2535 Edge 800 @@ -9647,13 +9843,14 @@ 0509 BT EDR Dongle 0706 PocketPC e740 0707 Pocket PC e330 Series - 0708 Pocket PC e350Â Series + 0708 Pocket PC e350 Series 0709 Pocket PC e750 Series 070a Pocket PC e400 Series 070b Pocket PC e800 Series 0a07 WLM-10U1 802.11abgn Wireless Adapter [Ralink RT3572] 0b05 PX1220E-1G25 External hard drive 0b09 PX1396E-3T01 External hard drive + 0b1a STOR.E ALU 2S 1300 Wireless Broadband (CDMA EV-DO) SM-Bus Minicard Status Port 1301 Wireless Broadband (CDMA EV-DO) Minicard Status Port 1302 Wireless Broadband (3G HSDPA) SM-Bus Minicard Status Port @@ -9716,6 +9913,7 @@ 0933 Quantum Corp. 0934 Spirent Communications 0936 NuTesla + 000c Rhythmedics 6 BioData Integrator 0030 Composite Device, Mass Storage Device (Flash Drive) amd HID 003c Rhythmedics HID Bootloader 0939 Lumberg, Inc. @@ -9732,6 +9930,7 @@ 2471 SoC PC-Camera 2500 USB Optical Mouse 2510 Optical Mouse + 2521 Optical Mouse 2600 Typhoon Easycam USB 330K (newer)/Typhoon Easycam USB 2.0 VGA 1.3M/Sansun SN-508 2601 SPC 610NC Laptop Camera 2603 PAC7312 Camera @@ -9812,6 +10011,7 @@ 0955 NVidia Corp. 7030 Tegra 3 (recovery mode) 7100 Notion Ink Adam + 7820 Tegra 2 AC100 developer mode b400 SHIELD (debug) b401 SHIELD 0956 BSquare Corp. @@ -9850,6 +10050,7 @@ 0977 Lightsurf Technologies 0978 Beckhoff GmbH 0979 Jeilin Technology Corp., Ltd + 0222 Keychain Display 0224 JL2005A Toy Camera 0226 JL2005A Toy Camera 0227 JL2005B/C/D Toy Camera @@ -9961,7 +10162,8 @@ 032b Wireless Mouse (Battery Free) 8090 X-718BK Oscar Optical Gaming Mouse 9033 X-718BK Optical Mouse - 9090 XL-750BK Laser Mouse + 9066 F3 V-Track Gaming Mouse + 9090 XL-730K / XL-750BK / XL-755BK Mice 09db Measurement Computing Corp. 0075 MiniLab 1008 0076 PMD-1024 @@ -10231,6 +10433,7 @@ 0009 LP2844 Printer 0081 GK420t Label Printer 008b HC100 wristbands Printer + 00d1 Zebra GC420d Label Printer 930a Printer 0a62 MPMan 0010 MPMan MP-F40 MP3 Player @@ -10408,6 +10611,7 @@ c303 Saturn USB 2.0 Camera c326 Namuga 1.3M Webcam c33f Webcam + c429 Lenovo ThinkCentre Web Camera 0ac9 Micro Solutions, Inc. 0000 Backpack CD-ReWriter 0001 BACKPACK 2 Cable @@ -10423,6 +10627,7 @@ 0300 IDT1221U RS-232 Adapter 0401 Spectrum III Hybrid Smartcard Reader 0630 Spectrum III Mag-Only Insert Reader (SPT3-355 Series) USB-CDC + 0810 SecurePIN (IDPA-506100Y) PIN Pad 0ace ZyDAS 1201 ZD1201 802.11b 1211 ZD1211 802.11g @@ -10557,8 +10762,10 @@ 0b0d ProjectLab 0000 CenturyCD 0b0e GN Netcom + 0420 Jabra SPEAK 510 1022 Jabra PRO 9450, Type 9400BS (DECT Headset) 620c Jabra BT620s + 9330 Jabra GN9330 Headset 0b0f AVID Technology 0b10 Pcally 0b11 I Tech Solutions Co., Ltd @@ -10660,6 +10867,10 @@ 0b63 ADLink Technology, Inc. 0b64 Wonderful Wire Cable Co., Ltd 0b65 Expert Magnetics Corp. +0b66 Cybiko Inc. + 0041 Xtreme +0b67 Fairbanks Scales + 555e SCB-R9000 0b69 CacheVision 0b6a Maxim Integrated Products a132 WUP-005 [Nintendo Wii U Pro Controller] @@ -10829,7 +11040,7 @@ 0a4d PocketPC Sync 0a4e PocketPC Sync 0a4f PocketPC Sync - 0a50 HTC SmartPhone Sync + 0a50 SmartPhone (MTP) 0a51 SPV C400 / T-Mobile SDA GSM/GPRS Pocket PC 0a52 SmartPhone Sync 0a53 SmartPhone Sync @@ -10952,9 +11163,11 @@ 0502 ST3300601CB-RK 300 GB External Hard Drive 0503 ST3250824A [Barracuda 7200.9] 2000 Storage Adapter V3 (TPP) + 2100 FreeAgent Go 2200 FreeAgent Go FW 2300 Expansion Portable 2320 USB 3.0 bridge [Portable Expansion Drive] + 3008 FreeAgent Desk 1TB 3320 SRD00F2 [Expansion Desktop Drive] 3332 Expansion 5021 FreeAgent GoFlex USB 2.0 @@ -10964,6 +11177,8 @@ 5121 FreeAgent GoFlex 5161 FreeAgent GoFlex dock a003 Backup Plus + a0a1 Backup Plus Desktop + a0a4 Backup Plus Desktop Drive 0bc3 IPWireless, Inc. 0001 UMTS-TDD (TD-CDMA) modem 0bc4 Microcube Corp. @@ -11032,6 +11247,7 @@ 8174 RTL8192SU 802.11n WLAN Adapter 8176 RTL8188CUS 802.11n WLAN Adapter 8178 RTL8192CU 802.11n WLAN Adapter + 8179 RTL8188EUS 802.11n Wireless Network Adapter 817f RTL8188RU 802.11n WLAN Adapter 8187 RTL8187 Wireless Adapter 8189 RTL8187B Wireless 802.11g 54Mbps Network Adapter @@ -11084,6 +11300,7 @@ 1009 Connect2Air E-5400 D1700 802.11g Wireless Adapter [Intersil ISL3887] 100c Keyboard FSC KBPC PX 100f miniCard D2301 802.11bg Wireless Module [SiS 163U] + 1017 Keyboard KB SCR 0bfd Kvaser AB 0004 USBcan II 000b Leaf Light HS @@ -11160,7 +11377,7 @@ 3bfa pcProx Card Reader 0c2e Metrologic Instruments 0007 Metrologic MS7120 Barcode Scanner (IBM SurePOS mode) - 0200 Metrologic Scanner + 0200 MS7120 Barcode Scanner 0204 Metrologic MS7120 Barcode Scanner (keyboard mode) 0206 Metrologic MS4980 Barcode Scanner 0700 Metrologic MS7120 Barcode Scanner (uni-directional serial mode) @@ -11303,12 +11520,14 @@ 62c0 Sonix USB 2.0 Camera 62e0 MSI Starcam Racer 6310 Sonix USB 2.0 Camera + 6341 Defender G-Lens 2577 HD720p Camera 63e0 Sonix Integrated Webcam 63f1 Integrated Webcam 63f8 Sonix Integrated Webcam 6409 Webcam 6413 Integrated Webcam 6417 Integrated Webcam + 6419 Integrated Webcam 641d 1.3 MPixel Integrated Webcam 6480 Sonix 1.3 MP Laptop Integrated Webcam 648b Integrated Webcam @@ -11467,7 +11686,7 @@ 3050 EZ710 Smart Card Reader 0ca7 Information Systems Laboratories 0cad Motorola CGISS - 9001 PowerPad Pocket PCÂ Device + 9001 PowerPad Pocket PC Device 0cae Ascom Business Systems, Ltd 0caf Buslink 2507 Hi-Speed USB-to-IDE Bridge Controller @@ -11518,7 +11737,7 @@ 0035 Miditech Play'n Roll 0036 Cinergy 250 Audio 0037 Cinergy 250 Audio - 0038 Cinergy T² DVB-T Receiver + 0038 Cinergy T² DVB-T Receiver 0039 Grabster AV 400 003b Cinergy 400 003c Grabster AV 250 @@ -11527,7 +11746,7 @@ 004e Cinergy T XS 004f Cinergy Analog XS 0055 Cinergy T XE (Version 1, AF9005) - 005c Cinergy T² + 005c Cinergy T² 0069 Cinergy T XE (Version 2, AF9015) 006b Cinergy HT PVR (EU) 0072 Cinergy Hybrid T @@ -11540,6 +11759,7 @@ 00a5 Cinergy Hybrid Stick 00a9 RTL2838 DVB-T COFDM Demodulator [TerraTec Cinergy T Stick Black] 00b3 NOXON DAB/DAB+ Stick + 00e0 NOXON DAB/DAB+ Stick V2 10a7 TerraTec G3 0cd4 Bang Olufsen 0101 BeolinkPC2 @@ -11591,7 +11811,7 @@ 0005 AR5523 0006 AR5523 (no firmware) 1001 Thomson TG121N [Atheros AR9001U-(2)NG] - 1002 TP-Link TL-WN821N v2 802.11n [Atheros AR9170] + 1002 TP-Link TL-WN821N v2 / TL-WN822N v1 802.11n [Atheros AR9170] 1006 TP-Link TL-WN322G v3 / TL-WN422G v2 802.11g [Atheros AR9271] 1010 3Com 3CRUSBN275 802.11abgn Wireless Adapter [Atheros AR9170] 20ff AR7010 (no firmware) @@ -11599,7 +11819,7 @@ 3002 AR3011 Bluetooth 3005 AR3011 Bluetooth 3008 Bluetooth (AR3011) - 7015 TP-Link TL-WN821N v3 802.11n [Atheros AR7010+AR9287] + 7015 TP-Link TL-WN821N v3 / TL-WN822N v2 802.11n [Atheros AR7010+AR9287] 9170 AR9170 802.11n 9271 AR9271 802.11n b002 Ubiquiti WiFiStation 802.11n [Atheros AR9271] @@ -11697,6 +11917,7 @@ 0d4f EADS Airbus France 0d50 Cleware GmbH 0011 USB-Temp2 Thermometer + 0040 F4 foot switch 0d51 Volex (Asia) Pte., Ltd 0d53 HMI Co., Ltd 0d54 Holon Corp. @@ -12215,6 +12436,7 @@ 1332 5 IMT 1416 32 IT 1417 A43 IT + 14ad 97 Titanium HD 150e 80 G9 0e7b On-Tech Industry Co., Ltd 0e7e Gmate, Inc. @@ -12400,6 +12622,7 @@ 0f52 Wing Key Electrical Co., Ltd 0f53 Dongguan White Horse Cable Factory, Ltd 0f54 Kawai Musical Instruments Mfg. Co., Ltd + 0101 MP6 Stage Piano 0f55 AmbiCom, Inc. 0f5c Prairiecomm, Inc. 0f5d NewAge International, LLC @@ -12498,6 +12721,7 @@ 0112 W995 Walkman Phone 015a Xperia Pro [Media Transfer Protocol] 0166 Xperia Mini Pro + 0167 ST15i (Xperia mini) 0169 Xperia S 0172 Xperia P 0177 Xperia Ion [Mass Storage] @@ -12513,9 +12737,12 @@ 3138 Xperia X10 mini pro 3149 Xperia X8 5177 Xperia Ion [Debug Mode] + 518c C1605 [Xperia E dual] MTD mode 614f Xperia X12 (debug mode) 6166 Xperia Mini Pro + 618c C1605 [Xperia E dual] MSC mode 715a Xperia Pro [Tethering] + 7166 Xperia Mini Pro (Tethering mode) 7177 Xperia Ion [Tethering] 8004 9000 Phone [Mass Storage] adde Boot loader @@ -12570,6 +12797,8 @@ 1008 Mini stick Suunto 0fd0 Tulip Computers B.V. 0fd1 Giant Electronics Ltd. +0fd2 Seac Banche + 0001 RDS 6000 0fd4 Tenovis GmbH & Co., KG 0fd5 Direct Access Technology, Inc. 0fd9 Elgato Systems GmbH @@ -12611,6 +12840,8 @@ 0ff7 CHI SHING Computer Accessories Co., Ltd 0ffc Clavia DMI AB 0021 Nord Stage 2 +0ffd EarlySense + ff00 OEM 0fff Aopen, Inc. 1000 Speed Tech Corp. 1001 Ritronics Components (S) Pte., Ltd @@ -12626,7 +12857,14 @@ 618f Ally/Optimus One 61c6 Vortex (msc) 61cc Optimus S + 61f1 Optimus Android Phone [LG Software mode] + 61f9 V909 G-Slate 61fc Optimus 3 + 61fe Optimus Android Phone [USB tethering mode] + 6300 Optimus Android Phone + 631c Optimus Android Phone [MTP mode] + 631e Optimus Android Phone [Camera/PTP mode] + 6356 Optimus Android Phone [Virtual CD mode] 6800 CDMA Modem 7000 LG LDP-7024D(LD)USB a400 Renoir (KC910) @@ -12698,6 +12936,7 @@ 0068 3,5'' HDD case MD-231 1038 Ideazon, Inc. 0100 Zboard + 1361 Sensei 1039 devolo AG 0824 1866 802.11bg [Texas Instruments TNETW1450] 2140 dsl+ 1100 duo @@ -12747,6 +12986,9 @@ 0009 RE-BL PlayStation 3 IR-to-Bluetooth converter 1050 Yubico.com 0010 Yubikey + 0110 Yubikey NEO OTP + 0111 Yubikey NEO OTP+CCID + 0211 Gnubby 1053 Immanuel Electronics Co., Ltd 1054 BMS International Beheer N.V. 5004 DSL 7420 Loader @@ -12761,7 +13003,8 @@ 0702 Passport External HDD 0704 Passport External HDD 070a My Passport Essential SE - 071a My Passport 1TB + 071a My Passport + 0730 My Passport 0740 My Passport 0742 My Passport Essential SE 0748 My Passport 1TB USB 3.0 @@ -12880,6 +13123,7 @@ 107f KidzMouse, Inc. 1082 Shin-Etsukaken Co., Ltd 1083 Canon Electronics, Inc. + 161b DR-2010C Scanner 162c P-150 Scanner 1084 Pantech Co., Ltd 108a Chloride Power Protection @@ -12951,6 +13195,7 @@ 8789 C8051F34x Extender & EDID MGR [EMX-DVI] 87be C8051F34x HDMI Audio Extractor [EMX-HD-AUD] 8863 C8051F34x Bootloader + 8897 C8051F38x HDMI Splitter [UHBX] ea60 CP210x UART Bridge / myAVR mySmartUSB light ea61 CP210x UART Bridge ea70 CP210x UART Bridge @@ -12963,6 +13208,7 @@ 1101 MP3 Player 10cd Kycon, Inc. 10ce Silicon Labs + 000e Shinko/Sinfonia CHC-S2145 ea6a MobiData EDGE USB Modem 10cf Velleman Components, Inc. 2011 R-Engine MPEG2 encoder/decoder @@ -12976,6 +13222,7 @@ 0301 CP42 - Communication Processor 10d4 Man Boon Manufactory, Ltd 10d5 Uni Class Technology Co., Ltd + 0004 PS/2 Converter 5552 KVM Human Interface Composite Device (Keyboard/Mouse ports) 55a2 2Port KVMSwitcher 10d6 Actions Semiconductor Co., Ltd @@ -13215,11 +13462,15 @@ 524e RoadMate 1475T 5260 Triton Handheld GPS Receiver (300/400/500/1500/2000) 1210 DigiTech + 0016 RP500 Guitar Multi-Effects Processor 001b RP155 Guitar Multi-Effects Processor 001c RP255 Guitar Multi-Effects Processor 121e Jungsoft Co., Ltd 3403 Muzio JM250 Audio Player 1223 SKYCABLE ENTERPRISE. CO., LTD. +1228 Datapaq Limited + 0012 Q18 Data Logger + 0015 TPaq21/MPaq21 Datalogger 1230 Chipidea-Microelectronica, S.A. 1233 Denver Electronics 5677 FUSB200 mp3 player @@ -13368,17 +13619,20 @@ 14c3 K5005 Vodafone LTE/UMTS/GSM Modem/Networkcard 14c8 K5005 Vodafone LTE/UMTS/GSM MOdem/Networkcard 14c9 K3770 3G Modem + 14cf K3772 14d1 K3770 3G Modem (Mass Storage Mode) 14f1 Gobi 3000 HSPA+ Modem 1501 Pulse 1505 E398 LTE/UMTS/GSM Modem/Networkcard - 1506 E398 LTE/UMTS/GSM Modem/Networkcard + 1506 Modem/Networkcard 150a E398 LTE/UMTS/GSM Modem/Networkcard 1520 K3765 HSPA 1521 K4505 HSPA+ + 155a R205 Mobile WiFi (CD-ROM mode) 1805 AT&T Go Phone U2800A phone 1c05 E173s 3G broadband stick (modem on) 1c0b E173s 3G broadband stick (modem off) + 1c20 R205 Mobile WiFi (Charging) 1d50 ET302s TD-SCDMA/TD-HSDPA Mobile Broadband 380b WiMAX USB modem(s) 12d2 LINE TECH INDUSTRIAL CO., LTD. @@ -13514,6 +13768,7 @@ 0011 VFS5011 Fingerprint Reader 0018 Fingerprint scanner 003c VFS471 Fingerprint Reader + 003d VFS491 138e Jungo LTD 9000 Raisonance S.A. STM32 ARM evaluation board 1390 TOMTOM B.V. @@ -13557,7 +13812,7 @@ 0030 Multimix 8 13b3 Nippon Dics Co., Ltd. 13ba PCPlay - 0001 König Electronic CMP-KEYPAD12 Numeric Keypad + 0001 König Electronic CMP-KEYPAD12 Numeric Keypad 0017 PS/2 Keyboard+Mouse Adapter 0018 Barcode PCP-BCG4209 13be Ricoh Printing Systems, Ltd. @@ -13661,6 +13916,9 @@ 1402 Bowe Bell & Howell 1403 Sitronix 0001 Digital Photo Frame +1409 IDS Imaging Development Systems GmbH + 1000 generic (firmware not loaded yet) + 1485 uEye UI1485 140e Telechips, Inc. b011 TCC780X-based player (USB Boot mode) b021 TCC77X-based players (USB Boot mode) @@ -13712,6 +13970,7 @@ 0007 Development board JTAG 1446 X.J.GROUP 6a73 Stamps.com Model 510 5LB Scale + 6a78 DYMO Endicia 75lb Digital Scale 1453 Radio Shack 4026 26-183 Serial Cable 1456 Extending Wire & Cable Co., Ltd. @@ -13750,6 +14009,7 @@ e02c Infrared Receiver e03a eHome Infrared Receiver e03c eHome Infrared Receiver + e03d 2 Channel Audio e03e Infrared Receiver [IR605A/Q] 147e Upek 1000 Biometric Touchchip/Touchstrip Fingerprint Sensor @@ -13844,6 +14104,8 @@ 14d8 JAMER INDUSTRIES CO., LTD. 14dd Raritan Computer, Inc. 1007 D2CIM-VUSB KVM connector +14e0 WiNRADiO Communications + 0501 WR-G528e 'CHEETAH' 14e1 Dialogue Technology Corp. 5000 PenMount 5000 Touch Controller 14e5 SAIN Information & Communications Co., Ltd. @@ -13886,6 +14148,7 @@ 152a Thesycon Systemsoftware & Consulting GmbH 152d JMicron Technology Corp. / JMicron USA Technology Corp. 0539 JMS539 SuperSpeed SATA II 3.0G Bridge + 0770 Alienware Integrated Webcam 2329 JM20329 SATA Bridge 2335 ATA/ATAPI Bridge 2336 Hard Disk Drive @@ -13906,6 +14169,7 @@ 0017 RZ01-0035 Laser Gaming Mouse [Imperator] 001c RZ01-0036 Optical Gaming Mouse [Abyssus] 0024 Razer Mamba + 002e RZ01-0058 Gaming Mouse [Naga] 0036 RZ01-0075, Gaming Mouse [Naga Hex] 0101 Copperhead Mouse 0102 Tarantula Keyboard @@ -13960,6 +14224,7 @@ 0820 SmartPocket Class Device 1598 Kunshan Guoji Electronics Co., Ltd. 15a2 Freescale Semiconductor, Inc. + 0038 9S08JS Bootloader 003b USB2CAN Application for ColdFire DEMOJM board 0042 OSBDM - Debug Port 004f i.MX28 SystemOnChip in RecoveryMode @@ -14035,6 +14300,7 @@ 2007 RSA SecurID (R) Authenticator 15e4 Numark 0024 Mixtrack + 0140 ION VCR 2 PC / Video 2 PC 15e8 SohoWare 9100 NUB100 Ethernet [pegasus] 9110 10/100 USB Ethernet @@ -14368,9 +14634,11 @@ 0504 RETRO Innovations ZoomFloppy 054b GrauTec ReelBox OLED Display (external) 05be EasyLogic Board + 06f9 Gabotronics Xminilab 0753 Digistump DigiSpark 075c AB-1.x UAC1 [Audio Widget] 075d AB-1.x UAC2 [Audio Widget] + 080a S2E1 Interface 16d3 Frontline Test Equipment, Inc. 16d5 AnyDATA Corporation 6202 CDMA/UMTS/GPRS modem @@ -14476,6 +14744,8 @@ aa11 Web Cam 1753 GERTEC Telecomunicacoes Ltda. c901 PPC900 Pinpad Terminal +1756 ENENSYS Technologies + 0006 DiviPitch 1759 LucidPort Technology, Inc. 1761 ASUSTek Computer, Inc. (wrong ID) 0b05 802.11n Network Adapter (wrong ID - swapped vendor and device) @@ -14519,8 +14789,11 @@ 0101 UB1 boundary microphone 0200 StudioDock monitors (internal hub) 0201 StudioDock monitors (audio) + 0210 StudioGT monitors 0301 Q2U handheld microphone with XLR 0302 GoMic compact condenser microphone + 0304 Q2U handheld mic with XLR + 0305 GoMic compact condenser mic 0310 Meteor condenser microphone 17a4 Concept2 0001 Performance Monitor 3 @@ -14532,6 +14805,8 @@ 0005 M-Bus Master MultiPort 250D 17b3 Grey Innovation 0004 Linux-USB Midi Gadget +17b5 Lunatone + 0010 MFT Sensor 17ba SAURIS GmbH 0001 SAU510-USB [no firmware] 0510 SAU510-USB and SAU510-USB plus JTAG Emulators @@ -14559,6 +14834,7 @@ 17d3 Korea Techtron Co., Ltd. 17e9 DisplayLink 0051 USB VGA Adaptor + 030b HP T100 0377 Plugable UD-160-A (M) 0378 Plugable UGA-2K-A 0379 Plugable UGA-125 @@ -14567,10 +14843,12 @@ 037c Plugable DC-125 037d Plugable USB2-HDMI-165 430a HP Port Replicator (Composite Device) + 4312 S2340T 17eb Cornice, Inc. 17ef Lenovo 1003 Integrated Smart Card Reader 1004 Integrated Webcam + 1008 Hub 100a ThinkPad Mini Dock Plus Series 3 3815 ChipsBnk 2GB USB Stick 4802 Lenovo Vc0323+MI1310_SOC Camera @@ -14585,8 +14863,10 @@ 4813 Integrated Webcam [R5U877] 4814 Integrated Webcam [R5U877] 4815 Integrated Webcam [R5U877] + 4816 Integrated Webcam 481c Integrated Webcam 481d Integrated Webcam + 6004 ISD-V4 Tablet Pen 6007 Smartcard Keyboard 6009 ThinkPad Keyboard with TrackPoint 6014 Mini Wireless Keyboard N5901 @@ -14632,6 +14912,7 @@ 1873 Navilock ee93 EasyLogger 187c Alienware Corporation + 0511 AlienFX Mobile lighting 0600 Dual Compatible Game Pad 187f Siano Mobile Silicon 0010 Stallar Board @@ -14657,7 +14938,7 @@ 0227 Pocket Hard Drive 022b Portable Hard Drive (Store'n'Go) 0237 Portable Harddrive (500 GB) - 0302 32GB Flash Drive + 0302 Flash Drive 18b1 Petalynx 0037 Maxter Remote Control 18b4 e3C Technologies @@ -14686,15 +14967,17 @@ 4e22 Nexus S (debug) 4e24 Nexus S (tether) 4e40 Nexus 7 (fastboot) - 4e41 ASUS Nexus 7 (MTP modus) + 4e41 Nexus 7 (MTP) 4e42 Nexus 7 (debug) - 4e43 ASUS Nexus 7 (PTP modus) - 4ee1 Nexus 4 + 4e43 Nexus 7 (PTP) + 4ee1 Nexus 4 / 10 4ee2 Nexus 4 (debug) 4ee3 Nexus 4 (tether) 4ee4 Nexus 4 (debug + tether) 7102 Toshiba Thrive tablet b004 Pandigital / B&N Novel 9" tablet + d109 LG G2x MTP + d10a LG G2x MTP (debug) 18d5 Starline International Group Limited 18d9 Kaba 01a0 B-Net 91 07 @@ -14725,10 +15008,13 @@ 190d Motorola GSG 1914 Alco Digital Devices Limited 1915 Nordic Semiconductor ASA + 000c Wireless Desktop nRF24L01 CX-1766 2233 Linksys WUSB11 v2.8 802.11b Adapter [Atmel AT76C505] 2234 Linksys WUSB54G v1 OEM 802.11g Adapter [Intersil ISL3886] 2235 Linksys WUSB54GP v1 OEM 802.11g Adapter [Intersil ISL3886] 2236 Linksys WUSB11 v3.0 802.11b Adapter [Intersil PRISM 3] +1923 FitLinxx + 0002 Personal SyncPoint 1926 NextWindow 0003 1900 HID Touchscreen 0006 1950 HID Touchscreen @@ -14758,6 +15044,7 @@ 0085 1950 HID Touchscreen 0086 1950 HID Touchscreen 0087 1950 HID Touchscreen + 0dc2 HID Touchscreen 192f Avago Technologies, Pte. 0000 Mouse 0416 ADNS-5700 Optical Mouse Controller (3-button) @@ -14789,6 +15076,7 @@ 0301 AudioBox 1951 Hyperstone AG 1953 Ironkey Inc. + 0202 S200 2GB Rev. 1 1954 Radiient Technologies 195d Itron Technology iONE 7002 Libra-Q11 IR remote @@ -14868,6 +15156,8 @@ 2000 MF627/MF628/MF628+/MF636+ HSDPA/HSUPA fff2 Gobi Wireless Modem (QDL mode) fff3 Gobi Wireless Modem +19db KFI Printers + 02f1 NAUT324C 19e1 WeiDuan Electronic Accessory (S.Z.) Co., Ltd. 19e8 Industrial Technology Research Institute 19ef Pak Heng Technology (Shenzhen) Co., Ltd. @@ -14878,6 +15168,7 @@ 19ff Dynex 0102 1.3MP Webcam 0201 Rocketfish Wireless 2.4G Laser Mouse + 0238 DX-WRM1401 Mouse 1a08 Bellwood International, Inc. 1a0a USB-IF non-workshop badd USB OTG Compliance test device @@ -14895,7 +15186,7 @@ 0802 Gamepad 1a36 Biwin Technology Ltd. 1a40 Terminus Technology Inc. - 0101 4-Port HUB + 0101 Hub 0201 FE 2.1 7-port Hub 1a41 Action Electronics Co., Ltd. 1a44 VASCO Data Security International @@ -14912,6 +15203,8 @@ 1a72 Physik Instrumente 1008 E-861 PiezoWalk NEXACT Controller 1a79 Bayer Health Care LLC + 6002 Contour + 7410 Contour Next 1a7b Lumberg Connect GmbH & Co. KG 1a7c Evoluent 0068 VerticalMouse 3 @@ -14944,6 +15237,8 @@ 1ab1 Rigol Technologies 0588 DS1000 SERIES 1acb Salcomp Plc +1acc Midiplus Co, Ltd. + 0103 AudioLink plus 4x4 2.9.28 1ad1 Desay Wire Co., Ltd. 1ad4 APS 0002 KM290-HRS @@ -15086,9 +15381,34 @@ 1b47 Energizer Holdings, Inc. 0001 CHUSB Duo Charger (NiMH AA/AAA USB smart charger) 1b48 Plastron Precision Co., Ltd. +1b52 ARH Inc. + 2101 FXMC Neural Network Controller + 2102 FXMC Neural Network Controller V2 + 2103 FXMC Neural Network Controller V3 + 4101 Passport Reader CLR device + 4201 Passport Reader PRM device + 4202 Passport Reader PRM extension device + 4203 Passport Reader PRM DSP device + 4204 Passport Reader PRMC device + 4205 Passport Reader CSHR device + 4206 Passport Reader PRMC V2 device + 4301 Passport Reader MRZ device + 4302 Passport Reader MRZ DSP device + 4303 Passport Reader CSLR device + 4401 Card Reader + 4501 Passport Reader RFID device + 4502 Passport Reader RFID AIG device + 6101 Neural Network Controller + 6202 Fingerprint Reader device + 6203 Fingerprint Scanner device + 8101 Camera V1 + 8102 Recovery / Camera V2 + 8103 Camera V3 1b59 K.S. Terminals Inc. 1b5a Chao Zhou Kai Yuan Electric Co., Ltd. 1b65 The Hong Kong Standards and Testing Centre Ltd. +1b71 Fushicai + 3002 USBTV007 Video Grabber [EasyCAP] 1b72 ATERGI TECHNOLOGY CO., LTD. 1b73 Fresco Logic 1000 xHC1 Controller @@ -15137,16 +15457,21 @@ 1bae Vuzix Corporation 0002 VR920 Immersive Eyewear 1bbb T & A Mobile Phones + 011e Alcatel One Touch L100V / Telekom Speedstick LTE II + f017 Alcatel One Touch L100V / Telekom Speedstick LTE II 1bc4 Ford Motor Co. 1bc5 AVIXE Technology (China) Ltd. -1bc7 Telit +1bc7 Telit Wireless Solutions 0020 HE863 0021 HE910 + 0023 HE910-D ECM 1003 UC864-E 1004 UC864-G 1005 CC864-DUAL 1006 CC864-SINGLE 1010 DE910-DUAL + 1011 CE910-DUAL + 1200 LE920 1bce Contac Cable Industrial Limited 1bcf Sunplus Innovation Technology Inc. 0007 Optical Mouse @@ -15156,6 +15481,7 @@ 0c31 SPIF30x Serial-ATA bridge 2885 ASUS Webcam 2888 HP Universal Camera + 2b83 Laptop Integrated Webcam FHD 1bd0 Hangzhou Riyue Electronic Co., Ltd. 1bd5 BG Systems, Inc. 1bde P-TWO INDUSTRIES, INC. @@ -15217,7 +15543,7 @@ 1c7a LighTuning Technology Inc. 0801 Fingerprint Reader 1c7b LUXSHARE PRECISION INDUSTRY (SHENZHEN) CO., LTD. -1c83 Schomäcker GmbH +1c83 Schomäcker GmbH 0001 RS150 V2 1c87 2N TELEKOMUNIKACE a.s. 1c88 Somagic, Inc. @@ -15241,6 +15567,7 @@ 1cbe Luminary Micro Inc. 00fd In-Circuit Debug Interface 00ff Stellaris ROM DFU Bootloader + 0166 CANAL USB2CAN 1cbf FORTAT SKYMARK INDUSTRIAL COMPANY 1cc0 PlantSense 1cca NextWave Broadband Inc. @@ -15296,6 +15623,8 @@ 000a Dream Cheeky Mailbox Friends Alert 000d Dream Cheeky Big Red Button 0013 Dream Cheeky LED Message Board +1d45 Touch + 1d45 Foxlink Optical touch sensor 1d4d PEGATRON CORPORATION 0002 Ralink RT2770/2720 802.11b/g/n Wireless LAN Mini-USB Device 000c Ralink RT3070 802.11b/g/n Wireless Lan USB Device @@ -15303,6 +15632,7 @@ 1d50 OpenMoko, Inc. 5119 GTA01/GTA02 U-Boot Bootloader 602b FPGALink + 6053 Darkgame Controller 1d57 Xenta 0005 Wireless Receiver (Keyboard and Mouse) 0006 Wireless Receiver (RC Laser Pointer) @@ -15325,6 +15655,8 @@ 0104 Multifunction Composite Gadget 0105 FunctionFS Gadget 0200 Qemu Audio Device +1d90 Citizen + 201e PPU-700 1de1 Actions Microelectronics Co. 1101 Generic Display Device (Mass storage mode) c101 Generic Display Device @@ -15344,7 +15676,7 @@ 0402 FTDI232 [EasyPort] 0403 FTDI232 [EasyPort Mini] 0404 FTDI232 [Netzteil-GL] - 0405 FTDI232 [MotorPrüfstand] + 0405 FTDI232 [MotorPrüfstand] 0406 STM32F103 [EasyKit] 0407 LPC2378 [Robotino] 0408 LPC2378 [Robotino-Arm] @@ -15368,6 +15700,7 @@ 2030 2030 USB Keyboard 1e68 TrekStor GmbH & Co. KG 001b DataStation maxi g.u + 0050 DataStation maxi light 1e71 NZXT 0001 Avatar Optical Mouse 1e74 Coby Electronics Corporation @@ -15396,12 +15729,17 @@ 1ee8 ONDA COMMUNICATION S.p.a. 0014 MT833UP 1ef6 EADS Deutschland GmbH + 2233 Cassidian NH90 STTE 5064 FDR Interface + 5523 Cassidian SSDC Adapter II + 5545 Cassidian SSDC Adapter III 5648 RIU CSMU/BSD 564a Cassidian RIU CSMU/BSD Simulator 1f28 Cal-Comp 0020 CDMA USB Modem A600 0021 CD INSTALLER USB Device +1f3a Onda (unverified) + efe8 V972 tablet in flashing mode 1f44 The Neat Company 0001 NM-1000 scanner 1f48 H-TRONIC GmbH @@ -15608,6 +15946,9 @@ 648b TEW-648UBM 802.11n 150Mbps Micro Wireless N Adapter [Realtek RTL8188CUS] 2101 ActionStar 0201 SIIG 4-to-2 Printer Switch +2149 Advanced Silicon S.A. + 211b Touchscreen Controller + 2703 TS58xxA/TC56xxA [CoolTouch] 2162 Creative (?) 2031 Network Blaster Wireless Adapter 500c DE5771 Modem Blaster @@ -15626,10 +15967,17 @@ 4050 AirStick joystick 2227 SAMWOO Enterprise 3105 SKYDATA SKD-U100 +2232 Silicon Motion + 1005 WebCam SCB-0385N + 1028 WebCam SC-03FFL11939N + 1029 WebCam SC-13HDL11939N + 1037 WebCam SC-03FFM12339N 2233 RadioShack Corporation 6323 USB Electronic Scale 2237 Kobo Inc. 4161 eReader White +228d 8D Technologies inc. + 0001 Terminal Bike Key Reader 22a6 Pie Digital, Inc. ffff PieKey "beta" 4GB model 4E4F41482E4F5247 (SM3251Q BB) 22b8 Motorola PCS @@ -16247,8 +16595,8 @@ 2448 82801 PCI Bridge 3100 PRO/DSL 3220 Modem - WAN 3101 PRO/DSL 3220 Modem - 3240 AnyPoint® 3240 Modem - WAN - 3241 AnyPoint® 3240 Modem + 3240 AnyPoint® 3240 Modem - WAN + 3241 AnyPoint® 3240 Modem 8602 Miniature Card Slot 9303 Intel 8x930Hx Hub 9500 CE 9500 DVB-T @@ -16898,7 +17246,7 @@ HUT 07 Keyboard 031 \ and | (Backslash and Bar) 032 # and ~ (Hash and Tilde, Non-US Keyboard near right shift) 033 ; and : (Semicolon and Colon) - 034 ´ and " (Accent Acute and Double Quotes) + 034 ´ and " (Accent Acute and Double Quotes) 035 ` and ~ (Accent Grace and Tilde) 036 , and < (Comma and Less) 037 . and > (Period and Greater) @@ -18059,4 +18407,4 @@ VT 0302 Sequential Media VT 0400 External Vendor Specific VT 0401 Composite Video VT 0402 S-Video -VT 0403 Component Video \ No newline at end of file +VT 0403 Component Video diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractUSB.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/UsbDeviceIdMapper.java similarity index 57% rename from RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractUSB.java rename to RecentActivity/src/org/sleuthkit/autopsy/recentactivity/UsbDeviceIdMapper.java index b9a40d6407..b7b8abc56d 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractUSB.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/UsbDeviceIdMapper.java @@ -38,14 +38,13 @@ import org.sleuthkit.autopsy.coreutils.PlatformUtil; /** * Loads a file that maps USB IDs to names of makes and models. Uses Linux USB info. - * This should be renamed because it isn't extracting. It's just mapping IDs to names. */ -class ExtractUSB { - private static final Logger logger = Logger.getLogger(ExtractUSB.class.getName()); +class UsbDeviceIdMapper { + private static final Logger logger = Logger.getLogger(UsbDeviceIdMapper.class.getName()); private HashMap devices; private static final String DataFile = "USB_DATA.txt"; - public ExtractUSB() { + public UsbDeviceIdMapper() { try { loadDeviceMap(); } catch (FileNotFoundException ex) { @@ -57,12 +56,12 @@ class ExtractUSB { } /** - * Example inputs: - * Vid_XXXX&Pid_XXXX - * @param dev + * Parses the passed in device ID and returns info that includes make and model. + * + * @param dev String to parse (i.e.: Vid_XXXX&Pid_XXXX) * @return */ - public USBInfo get(String dev) { + public USBInfo parseAndLookup(String dev) { String[] dtokens = dev.split("[_&]"); String vID = dtokens[1]; String pID; @@ -72,6 +71,7 @@ class ExtractUSB { pID = dtokens[3]; } String key = vID + pID; + key = key.toUpperCase(); if (!devices.containsKey(key)) { return new USBInfo(null, null); } else { @@ -80,62 +80,78 @@ class ExtractUSB { } /** - * Reads the USB file. Syntax of file: - * - * # vendor vendor_name - * # device device_name <-- single tab - * # interface interface_name <-- two tabs + * Reads the local USB txt file and stores in map. + * * @throws FileNotFoundException * @throws IOException */ private void loadDeviceMap() throws FileNotFoundException, IOException { devices = new HashMap<>(); - PlatformUtil.extractResourceToUserConfigDir(this.getClass(), DataFile); + PlatformUtil.extractResourceToUserConfigDir(this.getClass(), DataFile, false); try (Scanner dat = new Scanner(new FileInputStream(new java.io.File(PlatformUtil.getUserConfigDirectory() + File.separator + "USB_DATA.txt")))) { + /* Syntax of file: + * + * # vendor vendor_name + * # device device_name <-- single tab + * # interface interface_name <-- two tabs + */ String line = dat.nextLine(); while (dat.hasNext()) { - String dvc = ""; - if (!(line.startsWith("#") || (line.equals("")))) { - String[] tokens = line.split("[\\t\\s]+"); - String vID = tokens[0]; - for (int n = 1; n < tokens.length; n++) { - dvc += tokens[n] + " "; - } - String pID = vID + "0000"; - USBInfo info = new USBInfo(dvc, null); - - // make an entry with just the vendor ID - devices.put(pID, info); - - // get the later lines that have specific products - line = dat.nextLine(); - if (line.startsWith("\t")) { - while (dat.hasNext() && line.startsWith("\t")) { - tokens = line.split("[\\t\\s]+"); - pID = vID + tokens[1]; - String device = ""; - line = dat.nextLine(); - for (int n = 2; n < tokens.length; n++) { - device += tokens[n] + " "; - } - - info = new USBInfo(dvc, device); - //make an entry where the key is both the vendor and product IDs concatenated - devices.put(pID, info); - } - } - } else { + + // comments + if ((line.startsWith("#")) || (line.equals(""))) { line = dat.nextLine(); + continue; } + + // stop once we've hitten the part of the file that starts to talk about class types if (line.startsWith("C 00")) { return; } + + String dvc = ""; + String[] tokens = line.split("[\\t\\s]+"); + String vID = tokens[0]; + for (int n = 1; n < tokens.length; n++) { + dvc += tokens[n] + " "; + } + + // make an entry with just the vendor ID + String pID = vID + "0000"; + pID = pID.toUpperCase(); + USBInfo info = new USBInfo(dvc, null); + devices.put(pID, info); + + // parseAndLookup the later lines that have specific products + line = dat.nextLine(); + if (line.startsWith("\t")) { + while (dat.hasNext() && line.startsWith("\t")) { + tokens = line.split("[\\t\\s]+"); + + // make key based on upper case version of vendor and product IDs + pID = vID + tokens[1]; + pID = pID.toUpperCase(); + + String device = ""; + line = dat.nextLine(); + for (int n = 2; n < tokens.length; n++) { + device += tokens[n] + " "; + } + + info = new USBInfo(dvc, device); + + // store based on the previously generated key + devices.put(pID, info); + } + } } } } + /** + * Stores the vendor information about a USB device + */ public class USBInfo { - private String vendor; private String product; @@ -144,10 +160,18 @@ class ExtractUSB { product = prod; } + /** + * Get Vendor (make) information + * @return + */ public String getVendor() { return vendor; } + /** + * Get product (model) information + * @return + */ public String getProduct() { return product; } diff --git a/ScalpelCarver/src/org/sleuthkit/autopsy/scalpel/ScalpelCarverIngestModule.java b/ScalpelCarver/src/org/sleuthkit/autopsy/scalpel/ScalpelCarverIngestModule.java index 5a0bc208a6..7db7b56701 100644 --- a/ScalpelCarver/src/org/sleuthkit/autopsy/scalpel/ScalpelCarverIngestModule.java +++ b/ScalpelCarver/src/org/sleuthkit/autopsy/scalpel/ScalpelCarverIngestModule.java @@ -53,7 +53,7 @@ class ScalpelCarverIngestModule extends IngestModuleAdapter implements FileInges private static final Logger logger = Logger.getLogger(ScalpelCarverIngestModule.class.getName()); private final String MODULE_OUTPUT_DIR_NAME = "ScalpelCarver"; private String moduleOutputDirPath; - private String configFileName = "scalpel.conf"; + private final String configFileName = "scalpel.conf"; private String configFilePath; private boolean initialized = false; private ScalpelCarver carver; @@ -100,7 +100,7 @@ class ScalpelCarverIngestModule extends IngestModuleAdapter implements FileInges // copy the default config file to the user's home directory if one // is not already there try { - PlatformUtil.extractResourceToUserConfigDir(this.getClass(), configFileName); + PlatformUtil.extractResourceToUserConfigDir(this.getClass(), configFileName, false); } catch (IOException ex) { String message = "Could not obtain the path to the Scalpel configuration file."; logger.log(Level.SEVERE, message, ex); diff --git a/SevenZip/build.xml b/SevenZip/build.xml deleted file mode 100644 index e1933d01f0..0000000000 --- a/SevenZip/build.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - Builds, tests, and runs the project org.sleuthkit.autopsy.sevenzip. - - diff --git a/SevenZip/manifest.mf b/SevenZip/manifest.mf deleted file mode 100644 index 9999013410..0000000000 --- a/SevenZip/manifest.mf +++ /dev/null @@ -1,6 +0,0 @@ -Manifest-Version: 1.0 -OpenIDE-Module: org.sleuthkit.autopsy.sevenzip/1 -OpenIDE-Module-Implementation-Version: 4 -OpenIDE-Module-Localizing-Bundle: org/sleuthkit/autopsy/sevenzip/Bundle.properties - - diff --git a/SevenZip/nbproject/build-impl.xml b/SevenZip/nbproject/build-impl.xml deleted file mode 100644 index 061d48f66d..0000000000 --- a/SevenZip/nbproject/build-impl.xml +++ /dev/null @@ -1,45 +0,0 @@ - - - - - - - - - - - - - You must set 'suite.dir' to point to your containing module suite - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/SevenZip/nbproject/project.properties b/SevenZip/nbproject/project.properties deleted file mode 100644 index 98d7408800..0000000000 --- a/SevenZip/nbproject/project.properties +++ /dev/null @@ -1,8 +0,0 @@ -javac.source=1.7 -javac.compilerargs=-Xlint -Xlint:-serial -license.file=../LICENSE-2.0.txt -nbm.homepage=http://www.sleuthkit.org/autopsy/ -nbm.module.author=Brian Carrier -nbm.needs.restart=true -spec.version.base=1.0 - diff --git a/SevenZip/nbproject/project.xml b/SevenZip/nbproject/project.xml deleted file mode 100644 index 2420492066..0000000000 --- a/SevenZip/nbproject/project.xml +++ /dev/null @@ -1,64 +0,0 @@ - - - org.netbeans.modules.apisupport.project - - - org.sleuthkit.autopsy.sevenzip - - - - org.netbeans.api.progress - - - - 1 - 1.32.1 - - - - org.openide.util.lookup - - - - 8.19.1 - - - - org.openide.util - - - - 8.25.1 - - - - org.sleuthkit.autopsy.core - - - - 9 - 7.1 - - - - org.sleuthkit.autopsy.corelibs - - - - 3 - 1.1 - - - - - - ext/sevenzipjbinding.jar - release/modules/ext/sevenzipjbinding.jar - - - ext/sevenzipjbinding-AllPlatforms.jar - release/modules/ext/sevenzipjbinding-AllPlatforms.jar - - - - diff --git a/SevenZip/nbproject/suite.properties b/SevenZip/nbproject/suite.properties deleted file mode 100644 index 29d7cc9bd6..0000000000 --- a/SevenZip/nbproject/suite.properties +++ /dev/null @@ -1 +0,0 @@ -suite.dir=${basedir}/.. diff --git a/Timeline/build.xml b/Timeline/build.xml deleted file mode 100644 index 15db18b5b7..0000000000 --- a/Timeline/build.xml +++ /dev/null @@ -1,33 +0,0 @@ - - - - - - Builds, tests, and runs the project org.sleuthkit.autopsy.timeline. - - - - - - - - - - - - - - TSK_HOME: ${env.TSK_HOME} - - - - - - - - - - - - - diff --git a/Timeline/manifest.mf b/Timeline/manifest.mf deleted file mode 100644 index 6cc867f901..0000000000 --- a/Timeline/manifest.mf +++ /dev/null @@ -1,7 +0,0 @@ -Manifest-Version: 1.0 -OpenIDE-Module: org.sleuthkit.autopsy.timeline/1 -OpenIDE-Module-Layer: org/sleuthkit/autopsy/timeline/layer.xml -OpenIDE-Module-Localizing-Bundle: org/sleuthkit/autopsy/timeline/Bundle.properties -OpenIDE-Module-Requires: org.openide.windows.WindowManager -OpenIDE-Module-Implementation-Version: 3 - diff --git a/Timeline/nbproject/build-impl.xml b/Timeline/nbproject/build-impl.xml deleted file mode 100644 index 4babba24f7..0000000000 --- a/Timeline/nbproject/build-impl.xml +++ /dev/null @@ -1,45 +0,0 @@ - - - - - - - - - - - - - You must set 'suite.dir' to point to your containing module suite - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/Timeline/nbproject/project.properties b/Timeline/nbproject/project.properties deleted file mode 100644 index 8e20c77c1a..0000000000 --- a/Timeline/nbproject/project.properties +++ /dev/null @@ -1,6 +0,0 @@ -javac.source=1.7 -javac.compilerargs=-Xlint -Xlint:-serial -license.file=../LICENSE-2.0.txt -nbm.homepage=http://www.sleuthkit.org/autopsy/ -nbm.needs.restart=true -spec.version.base=1.0.0 diff --git a/Timeline/nbproject/project.xml b/Timeline/nbproject/project.xml deleted file mode 100644 index 352e0fd55f..0000000000 --- a/Timeline/nbproject/project.xml +++ /dev/null @@ -1,113 +0,0 @@ - - - org.netbeans.modules.apisupport.project - - - org.sleuthkit.autopsy.timeline - - - - org.netbeans.api.progress - - - - 1 - 1.32.1 - - - - org.netbeans.modules.settings - - - - 1 - 1.35.1 - - - - org.openide.actions - - - - 6.26.1 - - - - org.openide.awt - - - - 7.46.1 - - - - org.openide.dialogs - - - - 7.25.1 - - - - org.openide.modules - - - - 7.32.1 - - - - org.openide.nodes - - - - 7.28.1 - - - - org.openide.util - - - - 8.25.2 - - - - org.openide.util.lookup - - - - 8.15.2 - - - - org.openide.windows - - - - 6.55.2 - - - - org.sleuthkit.autopsy.core - - - - 9 - 7.0 - - - - org.sleuthkit.autopsy.corelibs - - - - 3 - 1.1 - - - - - - - diff --git a/Timeline/nbproject/suite.properties b/Timeline/nbproject/suite.properties deleted file mode 100644 index 29d7cc9bd6..0000000000 --- a/Timeline/nbproject/suite.properties +++ /dev/null @@ -1 +0,0 @@ -suite.dir=${basedir}/.. diff --git a/docs/doxygen/Doxyfile b/docs/doxygen/Doxyfile index 3ec84adc77..54d3c9cf03 100644 --- a/docs/doxygen/Doxyfile +++ b/docs/doxygen/Doxyfile @@ -32,7 +32,7 @@ PROJECT_NAME = "Autopsy" # This could be handy for archiving the generated documentation or # if some version control system is used. -PROJECT_NUMBER = 3 +PROJECT_NUMBER = 3.1 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer @@ -672,6 +672,7 @@ INPUT = main.dox \ modAdvanced.dox \ platformConcepts.dox \ regressionTesting.dox \ + native_libs.dox \ ../../Core/src \ ../../CoreLibs/src \ ../../ExifParser/src \ @@ -893,7 +894,7 @@ GENERATE_HTML = YES # If a relative path is entered the value of OUTPUT_DIRECTORY will be # put in front of it. If left blank `html' will be used as the default path. -HTML_OUTPUT = api-docs +HTML_OUTPUT = api-docs/3.1 # The HTML_FILE_EXTENSION tag can be used to specify the file extension for # each generated HTML page (for example: .htm,.php,.asp). If it is left blank diff --git a/docs/doxygen/modAdvanced.dox b/docs/doxygen/modAdvanced.dox index 13fcc87319..547ba827d5 100644 --- a/docs/doxygen/modAdvanced.dox +++ b/docs/doxygen/modAdvanced.dox @@ -10,7 +10,7 @@ These aren't really advanced, but you don't need to know them in detail when you Some modules may have configuration settings that uses can change. We recommend that you use the infrastructure provided by Autopsy and NetBeans to do this so that all module condiguration is done in a single place. -Note: This option panel applies to all module types. Ingest modules have a second type of option panel that can be accessed when a data source is added to a case. Refer to \ref ingestmodule_making_configuration for details on how to use those option panels. +Note: This option panel applies to all module types. Ingest modules have a second type of option panel that can be accessed when a data source is added to a case. Refer to \ref ingest_modules_making_options for details on how to use those option panels. To add a panel to the options menu, right click the module and choose New > Other. Under the Module Development category, select Options Panel and press Next. diff --git a/docs/doxygen/modDev.dox b/docs/doxygen/modDev.dox index 840ff123e3..337093b3df 100644 --- a/docs/doxygen/modDev.dox +++ b/docs/doxygen/modDev.dox @@ -40,7 +40,7 @@ The Autopsy modules are encapsulated inside of NetBeans modules. A NetBeans modu \subsection mod_dev_mod_nb Creating a NetBeans Module -If this is your first module, then you will need to make a NetBeans module. If you have already made an Autopsy module and are now working on a second one, you can consider adding it to your pevious NetBeans module. +If this is your first module, then you will need to make a NetBeans module. If you have already made an Autopsy module and are now working on a second one, you can consider adding it to your previous NetBeans module. To make a NetBeans module: - Open the NetBeans IDE and go to File -> New Project. @@ -52,12 +52,14 @@ To make a NetBeans module: After the module is created, you will need to do some further configuration. - Right click on the newly created module and choose "Properties". -- You will need to configure the module to be dependent on modules from within the Autopsy platform. Go to the "Libraries" area and choose "Add" in the "Module Dependencies" section. Choose the "Autopsy-core" library. You now have access to the Autopsy services. +- You will need to configure the module to be dependent on modules from within the Autopsy platform. Go to the "Libraries" area and choose "Add" in the "Module Dependencies" section. Choose the: +-- "Autopsy-core" library to get access to the Autopsy services. +-- "NetBeans Lookup" library so that your module can be discovered by Autopsy. - If you later determine that you need to pull in external JAR files, then you will use the "Wrapped Jar" section to add them in. - Note, you will also need to come back to this section if you update the platform. You may need to add a new dependency for the version of the Autopsy-core that comes with the updated platform. - Autopsy requires that all modules restart Autopsy after they are installed. Configure your module this way under Build -> Packaging. Check the box that says Needs Restart on Install. -You now have a NetBeans module that is using Autopsy as its build platform. That means you will have access to all of the services and utilities that Autopsy provides (such as \ref platform_details). +You now have a NetBeans module that is using Autopsy as its build platform. That means you will have access to all of the services and utilities that Autopsy provides (such as \ref services_page). \subsubsection mod_dev_mod_config_other Optional Settings diff --git a/docs/doxygen/modIngest.dox b/docs/doxygen/modIngest.dox index 2512a96c37..347d9c9a5a 100644 --- a/docs/doxygen/modIngest.dox +++ b/docs/doxygen/modIngest.dox @@ -1,265 +1,420 @@ /*! \page mod_ingest_page Developing Ingest Modules -\section ingestmodule_modules Ingest Module Basics +\section ingest_modules_getting_started Getting Started -This section tells you how to make an Ingest Module. Ingest modules -analyze data from a data source (a disk image or set of logical -files). They typically focus on a specific type of data analysis. -The modules are loaded each time that Autopsy starts. The user can -choose to enable each module when they add an image to the case. -It assumes you have already setup your development environment as -described in \ref mod_dev_page. +This page describes how to develop ingest modules. It assumes you have +already set up your development environment as described in \ref mod_dev_page. + +Ingest modules analyze data from a data source (e.g., a disk image or a folder +of logical files). There are two types of ingest modules in Autopsy: -First, you need to choose the type of Ingest Module. +- Data-source-level ingest modules +- File-level ingest modules -- Data Source-level modules are passed in a reference to a top-level data source, such as an Image or folder of logical files. -These modules may query the database for a small set of specific files. For example, a Windows registry module that runs on the hive files. It is interested in only a small subset of the hard drive files. +The difference between these two types of modules is that data-source-level modules are called once and passed in a reference to a data source to analyze and file-level ingest modules are called for each file and passed in the file to analyze. Here are some guidelines for choosing the type of your ingest module: -- File-level modules are passed in a reference to each file. -The Ingest Manager chooses which files to pass and when. -These modules are intended to analyze most of the files on the system -For example, a hash calculation module that reads in the content of every file. +- Your module should be a data-source-level ingest module if it only needs to +retrieve and analyze a small subset of the files present in a data source and it can find those files based on data in the database (such as file names). +For example, a Windows registry analysis module that only processes +registry hive files should be implemented as a data-source-level ingest module because there are only a few registry hives and we can find them by name. +- Your module should be a file-level ingest module if it analyzes most or all of +the files from a data source, one file at a time. If you cannot rely on finding a file based on its name, it will need to be a file-level ingest module. For example, a hash look up +module might process every file system file by looking up its hash in one or +more known file and known bad files hash sets (hash databases). + +As you will learn a little later in this guide, it is possible to package a +data-source-level ingest module and a file-level ingest module together. You +would do this when you need to work at both levels to get all of your analysis +done. The modules in such a pair will be enabled or disabled together and will +have common per ingest job and global settings. + +The text below will refer to example code in the org.sleuthkit.autopsy.ingest.examples package. +The sample modules don't do anything +particularly useful, but they can serve as templates for developing your own +ingest modules. -Refer to org.sleuthkit.autopsy.ingest.example for sample source code of dummy modules. -\section ingest_common Commonalities +\section ingest_modules_lifecycle Ingest Module Life Cycle -There are several things about these module types that are common and we'll outline those here. For both modules, you will extend an interface and implement some methods. +Before we dive into the details of creating a module, it is important to understand the life cycle of the module. Note that this life cycle is much different for Autopsy 3.1 modules than it was for Autopsy 3.0. This section only talks about 3.1 modules. -Refer to the documentation for each method for its use. -- org.sleuthkit.autopsy.ingest.IngestModuleAbstract.init() is invoked when an ingest session starts. -- org.sleuthkit.autopsy.ingest.IngestModuleAbstract.complete() is invoked when an ingest session completes. -- org.sleuthkit.autopsy.ingest.IngestModuleAbstract.stop() is invoked on a module when an ingest session is interrupted by the user or system. -- org.sleuthkit.autopsy.ingest.IngestModuleAbstract.getName() returns the name of the module. -- org.sleuthkit.autopsy.ingest.IngestModuleAbstract.getDescription() returns a short description of the module. -- org.sleuthkit.autopsy.ingest.IngestModuleAbstract.getVersion() returns the version of the module. +You will need to implement at least 2 interfaces to make an ingest module: +-# A factory class that will be created when Autopsy starts and will provide configuration panels to Autopsy and create instances of the ingest modules. +-# An ingest module class that will be instantiated by the factory when the ingest modules are run. A new instance of this will be created for each thread. + +Here is an example sequence of events. Details will be provided below. +-# User launches Autopsy and it looks for classes that implement the org.sleuthkit.autopsy.ingest.IngestModuleFactory interface. +-# Autopsy finds and creates an instance of your FooIngestModuleFactory class. +-# User adds a disk image. +-# Autopsy presents the list of available ingest modules to the user and uses the utility methods from FooIngestModuleFactory class to get the module's name, description, and configuration panels. +-# User enables your module (and others). +-# Autopsy uses FooIngestModuleFactory to create two instances of FooIngestModule (Autopsy is using two threads to process the files). +-# Autopsy starts up the module, calls its process method, and shuts it down when all of the data is analyzed. + -The process() method is invoked to analyze the data. This is where -the analysis is done. The specific method depends on the module -type; it is passed either a data source or a file to process. We'll -cover this in later sections. This method will post results to the -blackboard and with inbox messages to the user. +\section ingest_modules_implementing_ingestmodulefactory_basic Creating a Basic Ingest Module + +\subsection ingest_modules_implementing_basic_factory Basic Ingest Module Factory + +The first step to write an ingest module is to make its factory. There are three general types of things that a factory does: +-# Provides basic information such as the module's name, version, and description. (required) +-# Creates ingest modules. (required) +-# Provides panels so that the user can configure the module. (optional) + +This section covers the required parts of a basic factory so that we can make the ingest module. A later section (\ref ingest_modules_making_options) +covers how you can use the factory to provide options to the user. + +To make writing a simple factory easier, Autopsy provides an adapter class that implements the "optional" methods in the interface. +Our basic factory will use the adapter. + +-# Create a class either manually or using the NetBeans wizards. Edit the class to extend org.sleuthkit.autopsy.ingest.IngestModuleFactoryAdapter. NetBeans will likely complain that you have not implemented the necessary methods and you can use its "hints" to automatically generate stubs for them. +-# Use the documentation for the org.sleuthkit.autopsy.ingest.IngestModuleFactory interface for details on what each method needs to do. You can also refer to org.sleuthkit.autopsy.examples.SampleIngestModuleFactory as an example. +-# Add NetBeans annotations so that the module is found at run time: +\code +@ServiceProvider(service = IngestModuleFactory.class) +\endcode + +You will also need to import org.openide.util.lookup.ServiceProvider and add a dependency on the NetBeans Lookup +API module to the NetBeans module that contains your ingest module. + +At this point, you should be able to compile your NetBeans module and run it. When you add a data source, +you should see the module in the list of ingest modules. If you don't see it, double check that you extend +correct class and added the service provider annotation. -\section ingest_datasrc Data Source-level Modules -To make a data source-level module, make a new Java class either manually or using the NetBeans wizards. Edit the class to extend "org.sleuthkit.autopsy.ingest.IngestModuleDataSource". NetBeans will likely complain that you have not implemented the necessary methods and you can use its "hints" to automatically generate stubs for them. Use the documentation for the org.sleuthkit.autopsy.ingest.IngestModuleDataSource class for details on what each needs to do. -You can also refer to org.sleuthkit.autopsy.examples.SampleDataSourceIngestModule as an example module. +\subsection ingest_modules_implementing_ingestmodule Understanding the IngestModule Interface + +Data source and file ingest modules have similar APIs. The main difference is what data gets passed +to the methods. Let's first cover the common concepts. + +Both modules implement the org.sleuthkit.autopsy.ingest.IngestModule interface, which defines two methods to allocate and free resources: +- org.sleuthkit.autopsy.ingest.IngestModule.startUp() +- org.sleuthkit.autopsy.ingest.IngestModule.shutDown() + +Use the previous links to get the details of each method. The ingest modules will also have to implement a process() +method that will get passed in either a DataSource or a File. + +This section outlines the basic idea of modules and the org.sleuthkit.autopsy.ingest.IngestModule documentation should +be referred to for more details. + +- startUp() will be called before any data is analyzed to initialize and allocate resources, process() will then be called with data to analyze, and then shutDown() will be called to free resources and send ingest messages (see \ref ingest_modules_making_results). +- Any setup procedures that could fail should be done in startUp() so that it can throw an exception and cause the ingest job to stop and notify the user. +- startUp(), process(), and shutDown() will be called from a single thread. So, basic modules do not need to worry about thread safety if they allocate resources for each instance of the module. If the module wants to share resources between instances, then it is responsible for synchronizing the the shared resource. See org.sleuthkit.autopsy.examples.SampleFileIngestModule as an example that shares resources. -Data source-level ingest modules must find the files that they want to analyze. The best way to do that is using one of the findFiles() methods in org.sleuthkit.autopsy.casemodule.services.FileManager. See \ref mod_dev_other_services for more details. +The org.sleuthkit.autopsy.ingest.DataSourceIngestModule and org.sleuthkit.autopsy.ingest.FileIngestModule +interfaces both extend org.sleuthkit.autopsy.ingest.IngestModule. +For your convenience, an ingest module that does not require +initialization and/or clean up may extend the abstract +org.sleuthkit.autopsy.ingest.IngestModuleAdapter class to get default +"do nothing" implementations of these methods. -Example snippet of an ingest-level module process() method: +\subsection ingest_modules_implementing_datasourceingestmodule Creating a Data Source Ingest Module + +To create a data source ingest module: +-# Make a new Java class either manually or +using the NetBeans wizards. +-# Make the class implement +org.sleuthkit.autopsy.ingest.DataSourceIngestModule and optionally make it +extend org.sleuthkit.autopsy.ingest.IngestModuleAdapter. +-# The NetBeans IDE +will complain that you have not implemented one or more of the required methods. +You can use its "hints" to automatically generate stubs for the missing methods. Use this page and the +documentation for the org.sleuthkit.autopsy.ingest.IngestModule and +org.sleuthkit.autopsy.ingest.DataSourceIngestModule interfaces for guidance on +what each method needs to do. Or you can copy the code from +org.sleuthkit.autopsy.examples.SampleDataSourceIngestModule and use it as a +template for your module. + +All data source ingest modules must implement the single method defined by the +org.sleuthkit.autopsy.ingest.DataSourceIngestModule interface: + +- org.sleuthkit.autopsy.ingest.DataSourceIngestModule.process() + +The process() method is where all of the work of a data source ingest module is +done. It will be called exactly once between startUp() and shutDown(). The +process() method receives a reference to an org.sleuthkit.datamodel.Content object +and an org.sleuthkit.autopsy.ingest.DataSourceIngestModuleStatusHelper object. +The former is a representation of the data source. The latter should be used +by the module instance to be a good citizen within Autopsy as it does its +potentially long-running processing. + + +Note that data source ingest modules must find the files that they want to analyze. +The best way to do that is using one of the findFiles() methods of the +org.sleuthkit.autopsy.casemodule.services.FileManager class. See +\ref mod_dev_other_services for more details. + +The final step to getting the basic ingest module working is to configure your factory class to create instances of it. To do this, you will need to change the isDataSourceIngestModuleFactory() method to return true and have the createDataSourceIngestModule() method return a new instance of your ingest module. Both of these methods have default implementations in the IngestModuleFactoryAdapter that we used. Your factory should have code similar to: \code -@Override -public void process(Content dataSource, IngestDataSourceWorkerController controller) { - - //we have some number workunits / sub-tasks to execute - //in this case, we know the number of total tasks in advance - final int totalTasks = 12; + @Override + public boolean isDataSourceIngestModuleFactory() { + return true; + } - //initialize the overall image ingest progress - controller.switchToDeterminate(); - controller.progress(totalTasks); - - for(int subTask = 0; subTask < totalTasks; ++subTask) { - //add cancellation support - if (controller.isCancelled() ) { - break; // break out early to let the thread terminate - } - - //do the work - try { - //sub-task may add blackboard artifacts and create an inbox message - performSubTask(i); - } catch (Exception ex) { - logger.log(Level.WARNING, "Exception occurred in subtask " + subTask, ex); - } - - //update progress - controller.progress(i+1); - } -} + @Override + public DataSourceIngestModule createDataSourceIngestModule(IngestModuleIngestJobSettings ingestOptions) { + return new FooDataSourceIngestModule(); // replace this class name with the name of your class + } \endcode -\section ingest_file File-level Modules +\subsection ingest_modules_implementing_fileingestmodule Creating a File Ingest Module -To make a File-level module, make a new Java class either manually or using the NetBeans wizards. Edit the class to extend "org.sleuthkit.autopsy.ingest.IngestModuleAbstractFile". NetBeans will likely complain that you have not implemented the necessary methods and you can use its "hints" to automatically generate stubs for them. Use the method documentation in the org.sleuthkit.autopsy.ingest.IngestModuleAbstractFile class to fill in the details. -You can also refer to org.sleuthkit.autopsy.examples.SampleFileIngestModule as an example module. +To create a file ingest module: +-# Make a new Java class either manually or +using the NetBeans wizards. +-# Make the class implement +org.sleuthkit.autopsy.ingest.FileIngestModule and optionally make it +extend org.sleuthkit.autopsy.ingest.IngestModuleAdapter. +-# The NetBeans IDE +will complain that you have not implemented one or more of the required methods. +You can use its "hints" to automatically generate stubs for the missing methods. Use this page and the +documentation for the org.sleuthkit.autopsy.ingest.IngestModule and +org.sleuthkit.autopsy.ingest.FileIngestModule interfaces for guidance on what +each method needs to do. Or you can copy the code from +org.sleuthkit.autopsy.examples.SampleFileIngestModule and use it as a +template for your module. -Unlike Data Source-level modules, file-level modules are singletons. Only a single instance is created for all files. -The same file-level module instance will be used for files in different images and even different cases if new cases are opened. +All file ingest modules must implement the single method defined by the +org.sleuthkit.autopsy.ingest.FileIngestModule interface: -Every file-level module should support multiple init() -> process() -> complete(), and init() -> process() -> stop() invocations. It should also support init() -> complete() sequences. A new case could be open for each call of init(). +- org.sleuthkit.autopsy.ingest.FileIngestModule.process() -Currently (and this is likely to change in the future), File-level ingest modules are Singletons (meaning that only a single instance is created for the runtime of Autopsy). -You will need to implement a public static getDefault() method that returns a static instance of the module. Note that if you skip this step, you will not see an error until Autopsy tries to load your module and the log will say that it does not have a getDefault method. +The process() method is where all of the work of a file ingest module is +done. It will be called repeatedly between startUp() and shutDown(), once for +each file Autopsy feeds into the pipeline of which the module instance is a part. The +process() method receives a reference to a org.sleuthkit.datamodel.AbstractFile +object. -The implementation of this method is very standard, example: +The final step to getting the basic ingest module working is to configure your factory class to create instances of it. To do this, you will need to change the isFileIngestModuleFactory() method to return true and have the createFileIngestModule() method return a new instance of your ingest module. Both of these methods have default implementations in the IngestModuleFactoryAdapter that we used. Your factory should have code similar to: \code -public static synchronized MyIngestModule getDefault() { + @Override + public boolean isFileIngestModuleFactory() { + return true; + } - //defaultInstance is a private static class variable - if (defaultInstance == null) { - defaultInstance = new MyIngestModule(); - } - return defaultInstance; -} + @Override + public FileIngestModule createFileIngestModule(IngestModuleIngestJobSettings ingestOptions) { + return new FooFileIngestModule(); // replace this class name with the name of your class + } \endcode -You should also make the constructor private to ensure the singleton status. +\section ingest_modules_services Platform Services -As a result of the singleton design, init() will be called multiple times and even for different cases. Ensure that you update local member variables accordingly each time init() is called. Again, this design will likely change, but it is what it is for now. +The previous section will allow you to get a module up and running that will be passed in either a file or a data source to analyze. +This section covers how you get access to more data and how you can display data to the user. -\section ingestmodule_registration Module Registration +\subsection ingest_modules_services_ingest Ingest Services -Modules are automatically discovered if they implement the proper interface. -Currently, a restart of Autopsy is required after a module is installed before it is discovered. +The singleton instance of the org.sleuthkit.autopsy.ingest.IngestServices class +provides services tailored to the needs of ingest modules, and a module developer +should use these utilities to log errors, send messages, get the current case, +fire events, persist simple global settings, etc. Refer to the documentation +of the IngestServices class for method details. -By default, modules that do not come with a standard Autopsy installation will run after the standard modules. No order -is implied. This design will likely change in the future, but currently manual configuration is needed to enforce order. +\subsection ingest_modules_making_results Giving the User Feedback + +Ingest modules run in the background. There are three ways to send messages and +save results so that the user can see them: + +- Use the blackboard for long-term storage of analysis results. These results +will be displayed in the results tree. +- Use the ingest messages inbox to notify users of high-value analysis results +that were also posted to the blackboard. +- Use the logging and/or message box utilities for error messages. -There is an XML pipeline configuration that contains the standard modules and specifies the order that they are run in. -If you need to specify the order of modules, then they needed to be manually addded to this file in the correct order. -This file is the same format as The Sleuth Kit Framework configuration file. -Refer to http://sleuthkit.org/sleuthkit/docs/framework-docs/pipeline_config_page.html which is an official documentation -for the pipeline configuration schema. +\subsection ingest_modules_making_results_bb Posting Results to the Blackboard +The blackboard is used to store results so that they are displayed in the results tree. +See \ref platform_blackboard for details on posting results to it. -Autopsy will provide tools for reconfiguring the ingest pipeline in the near future, -and user/developer will be able to reload current view of discovered modules, -reorder modules in the pipeline and set their arguments using GUI. +The blackboard defines artifacts for specific data types (such as web bookmarks). +You can use one of the standard artifact types, create your own, or simply post text +as a org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE.TSK_TOOL_OUTPUT artifact. +The latter is much easier (for example, you can simply copy in the output from +an existing tool), but it forces the user to parse the output themselves. - -\section ingestmodule_services Ingest Services - -Class org.sleuthkit.autopsy.ingest.IngestServices provides services specifically for the ingest modules -and a module developer should use these utilities to send messages, get current case, etc. Refer to its documentation for method details. - -Remember, update references to IngestServices and Cases with each call to init() inside of the module. - -Module developers are encouraged to use Autopsy's org.sleuthkit.autopsy.coreutils.Logger -infrastructure to log errors to the Autopsy log. -The logger can also be accessed using the org.sleuthkit.autopsy.ingest.IngestServices class. - -Certain modules may need need a persistant store (other than for storing results) for storing and reading -module configurations or state. -The ModuleSettings API can be used also via org.sleuthkit.autopsy.ingest.IngestServices class. - - -\section ingestmodule_making_results Making Results Available to User - -Ingest modules run in the background. There are three ways to send messages and save results so that the user can see them: -- Blackboard for long-term storage of analysis results and to display in the results tree. -- Ingest Inbox to notify user of high-value analysis results that were also posted to blackboard. -- Error messages. - -\subsection ingestmodule_making_results_bb Posting Results to Blackboard -The blackboard is used to store results so that they are displayed in the results tree. See \ref platform_blackboard for details on posting results to it. - -The blackboard defines artifacts for specific data types (such as web bookmarks). You can use one of the standard artifact types, create your own, or simply post text with a org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE.TSK_TOOL_OUTPUT. The later is much easier (for example, you can simply copy in the output from an existing tool), but it forces the user to parse the output themselves. - -When modules add data to the blackboard, -they should notify listeners of the new data by -invoking IngestServices.fireModuleDataEvent() method. +When modules add data to the blackboard, they should notify listeners of the new +data by invoking the org.sleuthkit.autopsy.ingest.IngestServices.fireModuleDataEvent() method. Do so as soon as you have added an artifact to the blackboard. -This allows other modules (and the main UI) to know when to query the blackboard for the latest data. -However, if you are writing a larger number of blackboard artifacts in a loop, it is better to invoke -IngestServices.fireModuleDataEvent() only once after the bulk write, not to flood the system with events. +This allows other modules (and the main UI) to know when to query the blackboard +for the latest data. However, if you are writing a large number of blackboard +artifacts in a loop, it is better to invoke org.sleuthkit.autopsy.ingest.IngestServices.fireModuleDataEvent() +only once after the bulk write, so as not to flood the system with events. -\subsection ingestmodule_making_results_inbox Posting Results to Message Inbox +\subsection ingest_modules_making_results_inbox Posting Results to the Message Inbox -Modules should post messages to the inbox when interesting data is found -that has also been posted to the blackboard. The idea behind these -messages are that they are presented in chronological order so that +Modules should post messages to the inbox when interesting data is found. +Of course, such data should also be posted to the blackboard as described above. The idea behind +the ingest messages is that they are presented in chronological order so that users can see what was found while they were focusing on something else. -Error messages are also sent here as is summary information after the module has run to give the user some feedback. + +Inbox messages should only be sent if the result has a low false positive rate +and will likely be relevant. For example, the core Autopsy hash lookup module +sends messages if known bad (notable) files are found, but not if known good +(NSRL) files are found. This module also provides a global setting +(using its global settings panel) that allows a user to turn these messages on +or off. + +Messages are created using the org.sleuthkit.autopsy.ingest.IngestMessage class +and posted to the inbox using the org.sleuthkit.autopsy.ingest.IngestServices.postMessage() +method. -These messages should only be sent if the result has a low false positive rate and will likely be relevant. -For example, the hash lookup module will send messages if known bad (notable) files are found, -but not if known good (NSRL) files are found. You can provide options to the users on when to make messages. +\subsection ingest_modules_making_results_error Reporting Errors - -A single message includes the module name, message subject, message details, -a unique message id (in the context of the originating module), and a uniqueness attribute. -The uniqueness attribute is used to group similar messages together -and to determine the overall importance priority of the message -(if the same message is seen repeatedly, it is considered lower priority). - -For example, for a keyword search module, the uniqueness attribute would the keyword that was hit. - -Messages are created using the org.sleuthkit.autopsy.ingest.IngestMessage class and posted to the inbox using org.sleuthkit.autopsy.ingest.IngestServices.postMessage() method. - - -\subsection ingestmodule_making_results_error Reporting Errors - -When an error occurs, you should send a message to the ingest inbox with an error level. The downside of this though is that the ingest inbox was not entirely designed for this goal and it is easy for the user to miss these messages. Therefore, we identify these messages in the IngestInbox and also post a pop-up message that comes up in the lower right. - -You can make your own message in the lower right by using -org.sleuthkit.autopsy.coreutils.MessageNotifyUtil.Notify.show() +When an error occurs, you should write an error message to the Autopsy logs, using a +logger obtained from org.sleuthkit.autopsy.ingest.IngestServices.getLogger(). +You could also send an error message to the ingest inbox. The +downside of this is that the ingest inbox was not really designed for this +purpose and it is easy for the user to miss these messages. Therefore, it is +preferable to post a pop-up message that is displayed in the lower right hand +corner of the main window by calling +org.sleuthkit.autopsy.coreutils.MessageNotifyUtil.Notify.show(). -\section ingestmodule_making_configuration Module Configuration - -Ingest modules may require user configuration. In \ref mod_dev_adv_options, you wll learn about Autopsy-wide settings. There are some -settings that are specific to ingest modules as well. - -The framework -supports two levels of configuration: simple and advanced. Simple settings enable the user to enable and disable basic things at run-time (using check boxes and such). -Advanced settings require more in-depth configuration with more powerful interface. - -As an example, the advanced configuration for the keyword search module allows you to add and create keyword lists, choose encodings, etc. The simple interface allows -you to enable and disable lists. - -Module configuration is module-specific: every module maintains its own configuration state and is responsible for implementing the graphical interface. -If a module needs simple or advanced configuration, it needs to implement methods in its interface. -The org.sleuthkit.autopsy.ingest.IngestModuleAbstract.hasSimpleConfiguration(), -org.sleuthkit.autopsy.ingest.IngestModuleAbstract.getSimpleConfiguration(), and org.sleuthkit.autopsy.ingest.IngestModuleAbstract.saveSimpleConfiguration() -methods should be used for simple configuration. This panel will be shown when the user chooses which ingest modules to enable. - -The advanced configuration is implemented with the -org.sleuthkit.autopsy.ingest.IngestModuleAbstract.hasAdvancedConfiguration(), -org.sleuthkit.autopsy.ingest.IngestModuleAbstract.getAdvancedConfiguration(), and -org.sleuthkit.autopsy.ingest.IngestModuleAbstract.saveAdvancedConfiguration() -methods. This panel can be accessed from the "Advanced" button when the user chooses which ingest modules to enable. -It is recommended that the advanced panel be the same panel that is used in the Options area (see \ref mod_dev_adv_options). - -Refer to \ref mod_dev_adv_properties for details on saving properties from these panels. - + +\subsection ingest_modules_making_options_ingest Ingest Job Options + +To provide options for each ingest job: +- hasIngestJobSettingsPanel() must return true +- getIngestJobSettingsPanel() must return a IngestModuleIngestJobSettingsPanel that displays the needed configuration options and returns a IngestModuleIngestJobSettings object based on the settings. +- You are free to implement IngestModuleIngestJobSettings and store whatever you want in it (as long as it is serializable) +- The IngestModuleIngestJobSettings object that was created during configuration will be passed back to the factory with each call to createDataSourceIngestModule() or createFileIngestModule(). The factory should cast it to its internal class that implements IngestModuleIngestJobSettings and pass that object into the constructor of its ingest module so that it can use the settings when it runs. + + +You can also implement the getDefaultIngestJobSettings() method to return the default settings that Autopsy should use when the module has not been run before. + +NOTE: We recommend storing simple data in the IngestModuleIngestJobSettings-based class. In the case of our hash lookup module, we store the string names of the hash databases to lookup in. We then get the hash database handles in the call to startUp() using the global module settings. + + +\subsection ingest_modules_making_options_global Global Options + +To provide global options: +- hasGlobalSettingsPanel() must return true +- getGlobalSettingsPanel() must return a org.sleuthkit.autopsy.ingest.IngestModuleGlobalSetttingsPanel with widgets to support the global settings. +- You are responsible for persisting global settings and may use the module +settings methods provided by org.sleuthkit.autopsy.ingest.IngestServices for +saving simple properties, or the facilities of classes such as +org.sleuthkit.autopsy.coreutils.PlatformUtil and org.sleuthkit.autopsy.coreutils.XMLUtil +for more sophisticated approaches. +- You are responsible for providing a way for the ingest module to obtain the global settings. For +example, the Autopsy core hash look up module comes with a singleton hash databases +manager. Users import and create hash databases using the global settings panel. +Then they select which hash databases to use for a particular job using the +ingest job settings panel. When a module instance runs, it gets the relevant +databases from the hash databases manager. +- You are responsible for having the ingest job options panel update itself if the global settings change (i.e. if a new item is added that must be listed on the ingest panel). + + + + +\section ingest_modules_pipeline_configuration Controlling the Ordering of Ingest Modules in Ingest Pipelines + +By default, ingest modules that are not part of the standard Autopsy +installation will run after the core ingest modules. No order is implied. This +will likely change in the future, but currently manual configuration is needed +to enforce sequencing of ingest modules. + +There is an ingest pipeline configuration XML file that specifies the order for +running the core ingest modules. If you need to insert your ingest modules in +the sequence of core modules or control the ordering of non-core modules, you +must edit this file by hand. You will find it in the config directory of your +Autopsy installation, typically something like "C:\Users\yourUserName\AppData\Roaming\.autopsy\dev\config\pipeline_config.xml" +on a Microsoft Windows platform. Check the Userdir listed in the Autopsy About +dialog. + +Autopsy will provide tools for reconfiguring the ingest pipeline in the near +future. Until that time, there is no guarantee that the schema of this file will +remain fixed and that it will not be overwritten when upgrading your Autopsy +installation. + +\section ingest_modules_api_migration Migrating Ingest Modules to the Current API + +This section is a guide for module developers who wrote modules for the 3.0 API. These API changes occurred so that +we could make parallel pipelines of the file-level ingest modules. This section assumes you've read the above description of the new API. + +There are three big changes to make in your module: +-# Modules are no longer singletons. Autopsy will make one of your factory classes and many instances of the ingest modules. As part of the migration to the new classes, your singleton infrastructure will disappear. +-# You'll need to move the UI/Configuration methods into the factory class and the ingest module methods into their own class. You'll also need to update the APIs for the methods a bit. +-# You'll need to review your ingest module code for thread safety if you are using any static member variables. + + +We recommend that you: +-# Create a new factory class and move over the UI panels, configuration code, and standard methods (name, description, version, etc.). +-- You'll probably want the name in the ingest module code, so you should also store the name in a package-wide member variable. +-# Get the factory to compile and work. You can do basic testing by running Autopsy and verifying that you see your module and its panels. +-# Change your old ingest module to implement the new interface and adjust it (see the name changes below). Then update the factory to create it. +-# Review the ingest module code for thread safety (especially look for static member variables) + + +The following tables provide a mapping of the methods of the old abstract classes to +the new interfaces: + +Old method | New Method | +---------- | ---------- | +IngestModuleAbstract.getType() | N/A | +IngestModuleAbstract.init() | IngestModule.startUp() | +IngestModuleAbstract.getName() | IngestModuleFactory.getModuleName() | +IngestModuleAbstract.getDescription() | IngestModuleFactory.getModuleDescription() | +IngestModuleAbstract.getVersion() | IngestModuleFactory.getModuleVersion() | +IngestModuleAbstract.hasBackgroundJobsRunning | N/A | +IngestModuleAbstract.complete() | IngestModule.shutDown() | +IngestModuleAbstract.hasAdvancedConfiguration() | IngestModuleFactory.hasGlobalSettingsPanel() | +IngestModuleAbstract.getAdvancedConfiguration() | IngestModuleFactory.getGlobalSettingsPanel() | +IngestModuleAbstract.saveAdvancedConfiguration() | IngestModuleGlobalSetttingsPanel.saveSettings() | +N/A | IngestModuleFactory.getDefaultIngestJobSettings() | +IngestModuleAbstract.hasSimpleConfiguration() | IngestModuleFactory.hasIngestJobSettingsPanel() | +IngestModuleAbstract.getSimpleConfiguration() | IngestModuleFactory.getIngestJobSettingsPanel() | +IngestModuleAbstract.saveSimpleConfiguration() | N/A | +N/A | IngestModuleIngestJobSettingsPanel.getSettings() | +N/A | IngestModuleFactory.isDataSourceIngestModuleFactory() | +N/A | IngestModuleFactory.createDataSourceIngestModule() | +N/A | IngestModuleFactory.isFileIngestModuleFactory() | +N/A | IngestModuleFactory.createFileIngestModule() | + +Notes: +- IngestModuleFactory.getModuleName() should delegate to a static class method +that can also be called by ingest module instances. +- Autopsy passes a flag to IngestModule.shutDown() indicating whether the ingest +job completed or was cancelled. +- The global settings panel (formerly "advanced") for a module must implement +IngestModuleGlobalSettingsPanel which extends JPanel. Global settings are those +that affect all modules, regardless of ingest job and pipeline. +- The per ingest job settings panel (formerly "simple") for a module must implement +IngestModuleIngestJobSettingsPanel which extends JPanel. It takes the settings +for the current context as a serializable IngestModuleIngestJobSettings object +and its getSettings() methods returns a serializable IngestModuleIngestJobSettings object. +The IngestModuleIngestJobSettingsPanel.getSettings() method replaces the saveSimpleSettings() method, +except that now Autopsy persists the settings in a context-sensitive fashion. +- The IngestModuleFactory creation methods replace the getInstance() methods of +the former singletons and receive a IngestModuleIngestJobSettings object that should be +passed to the constructors of the module instances the factory creates. */ diff --git a/docs/doxygen/services.dox b/docs/doxygen/services.dox index 48b8039025..ae6c293d03 100644 --- a/docs/doxygen/services.dox +++ b/docs/doxygen/services.dox @@ -24,7 +24,7 @@ The followig are basic services that are available. - FileManager: the org.sleuthkit.autopsy.casemodule.services.FileManager service provides an API to access any file in the case. You can access FileManager by calling org.sleuthkit.autopsy.casemodule.services.Services.getFileManager(). Data Source-level Ingest modules and Report modules typically use this service because the other modules are passed in a reference to a specific file to do something with. - org.sleuthkit.autopsy.coreutils.Logger - Use this class to log error and informational messages to the central Autopsy log file. - If you have a background task that needs the provide the user with feedback, you can use the org.sleuthkit.autopsy.coreutils.MessageNotifyUtil.Notify.show() method to make a message in the lower right hand area. -- IngestModules also have a class that provides additional services. See \ref ingestmodule_services. +- IngestModules also have a class that provides additional services. See \ref ingest_modules_services_ingest. \subsection mod_dev_other_utilities Framework Utilities diff --git a/ewfVerify/src/org/sleuthkit/autopsy/ewfverify/EwfVerifyIngestModule.java b/ewfVerify/src/org/sleuthkit/autopsy/ewfverify/EwfVerifyIngestModule.java index e0c7819377..5d039fe304 100755 --- a/ewfVerify/src/org/sleuthkit/autopsy/ewfverify/EwfVerifyIngestModule.java +++ b/ewfVerify/src/org/sleuthkit/autopsy/ewfverify/EwfVerifyIngestModule.java @@ -24,7 +24,7 @@ import java.util.logging.Level; import java.util.logging.Logger; import javax.xml.bind.DatatypeConverter; import org.sleuthkit.autopsy.ingest.IngestModuleAdapter; -import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleStatusHelper; +import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProgress; import org.sleuthkit.autopsy.ingest.IngestMessage; import org.sleuthkit.autopsy.ingest.IngestMessage.MessageType; import org.sleuthkit.autopsy.ingest.IngestServices; @@ -49,17 +49,18 @@ public class EwfVerifyIngestModule extends IngestModuleAdapter implements DataSo private Image img; private String imgName; private MessageDigest messageDigest; - private static int messageId = 0; // RJCTODO: Not thread safe private boolean verified = false; private boolean skipped = false; private String calculatedHash = ""; private String storedHash = ""; + private IngestJobContext context; EwfVerifyIngestModule() { } @Override public void startUp(IngestJobContext context) throws IngestModuleException { + this.context = context; verified = false; skipped = false; img = null; @@ -80,14 +81,14 @@ public class EwfVerifyIngestModule extends IngestModuleAdapter implements DataSo } @Override - public ProcessResult process(Content dataSource, DataSourceIngestModuleStatusHelper statusHelper) { + public ProcessResult process(Content dataSource, DataSourceIngestModuleProgress statusHelper) { imgName = dataSource.getName(); try { img = dataSource.getImage(); } catch (TskCoreException ex) { img = null; logger.log(Level.SEVERE, "Failed to get image from Content.", ex); - services.postMessage(IngestMessage.createMessage(++messageId, MessageType.ERROR, EwfVerifierModuleFactory.getModuleName(), + services.postMessage(IngestMessage.createMessage( MessageType.ERROR, EwfVerifierModuleFactory.getModuleName(), NbBundle.getMessage(this.getClass(), "EwfVerifyIngestModule.process.errProcImg", imgName))); @@ -98,7 +99,7 @@ public class EwfVerifyIngestModule extends IngestModuleAdapter implements DataSo if (img.getType() != TskData.TSK_IMG_TYPE_ENUM.TSK_IMG_TYPE_EWF_EWF) { img = null; logger.log(Level.INFO, "Skipping non-ewf image {0}", imgName); - services.postMessage(IngestMessage.createMessage(++messageId, MessageType.INFO, EwfVerifierModuleFactory.getModuleName(), + services.postMessage(IngestMessage.createMessage( MessageType.INFO, EwfVerifierModuleFactory.getModuleName(), NbBundle.getMessage(this.getClass(), "EwfVerifyIngestModule.process.skipNonEwf", imgName))); @@ -110,7 +111,7 @@ public class EwfVerifyIngestModule extends IngestModuleAdapter implements DataSo storedHash = img.getMd5().toLowerCase(); logger.log(Level.INFO, "Hash value stored in {0}: {1}", new Object[]{imgName, storedHash}); } else { - services.postMessage(IngestMessage.createMessage(++messageId, MessageType.ERROR, EwfVerifierModuleFactory.getModuleName(), + services.postMessage(IngestMessage.createMessage( MessageType.ERROR, EwfVerifierModuleFactory.getModuleName(), NbBundle.getMessage(this.getClass(), "EwfVerifyIngestModule.process.noStoredHash", imgName))); @@ -118,7 +119,7 @@ public class EwfVerifyIngestModule extends IngestModuleAdapter implements DataSo } logger.log(Level.INFO, "Starting hash verification of {0}", img.getName()); - services.postMessage(IngestMessage.createMessage(++messageId, MessageType.INFO, EwfVerifierModuleFactory.getModuleName(), + services.postMessage(IngestMessage.createMessage( MessageType.INFO, EwfVerifierModuleFactory.getModuleName(), NbBundle.getMessage(this.getClass(), "EwfVerifyIngestModule.process.startingImg", imgName))); @@ -126,7 +127,7 @@ public class EwfVerifyIngestModule extends IngestModuleAdapter implements DataSo long size = img.getSize(); if (size == 0) { logger.log(Level.WARNING, "Size of image {0} was 0 when queried.", imgName); - services.postMessage(IngestMessage.createMessage(++messageId, MessageType.ERROR, EwfVerifierModuleFactory.getModuleName(), + services.postMessage(IngestMessage.createMessage( MessageType.ERROR, EwfVerifierModuleFactory.getModuleName(), NbBundle.getMessage(this.getClass(), "EwfVerifyIngestModule.process.errGetSizeOfImg", imgName))); @@ -146,7 +147,7 @@ public class EwfVerifyIngestModule extends IngestModuleAdapter implements DataSo // Read in byte size chunks and update the hash value with the data. for (int i = 0; i < totalChunks; i++) { - if (statusHelper.isIngestJobCancelled()) { + if (context.isJobCancelled()) { return ProcessResult.OK; } data = new byte[(int) chunkSize]; @@ -155,7 +156,7 @@ public class EwfVerifyIngestModule extends IngestModuleAdapter implements DataSo } catch (TskCoreException ex) { String msg = NbBundle.getMessage(this.getClass(), "EwfVerifyIngestModule.process.errReadImgAtChunk", imgName, i); - services.postMessage(IngestMessage.createMessage(++messageId, MessageType.ERROR, EwfVerifierModuleFactory.getModuleName(), msg)); + services.postMessage(IngestMessage.createMessage( MessageType.ERROR, EwfVerifierModuleFactory.getModuleName(), msg)); logger.log(Level.SEVERE, msg, ex); return ProcessResult.ERROR; } @@ -179,7 +180,7 @@ public class EwfVerifyIngestModule extends IngestModuleAdapter implements DataSo extra += "
  • Result:" + msg + "
  • "; extra += "
  • Calculated hash: " + calculatedHash + "
  • "; extra += "
  • Stored hash: " + storedHash + "
  • "; - services.postMessage(IngestMessage.createMessage(++messageId, MessageType.INFO, EwfVerifierModuleFactory.getModuleName(), imgName + msg, extra)); + services.postMessage(IngestMessage.createMessage( MessageType.INFO, EwfVerifierModuleFactory.getModuleName(), imgName + msg, extra)); logger.log(Level.INFO, "{0}{1}", new Object[]{imgName, msg}); } } diff --git a/git-daemon-export-okay b/git-daemon-export-okay new file mode 100644 index 0000000000..e69de29bb2 diff --git a/nbproject/project.properties b/nbproject/project.properties index f6595e2a4f..73cc3f420b 100644 --- a/nbproject/project.properties +++ b/nbproject/project.properties @@ -9,7 +9,6 @@ app.version=3.1.0_Beta ### Must be one of: DEVELOPMENT, RELEASE build.type=RELEASE project.org.sleuthkit.autopsy.ewfverify=ewfVerify -project.org.sleuthkit.autopsy.fileextmismatch=FileExtMismatch #build.type=DEVELOPMENT update_versions=false #custom JVM options @@ -28,15 +27,10 @@ modules=\ ${project.org.sleuthkit.autopsy.recentactivity}:\ ${project.org.sleuthkit.autopsy.testing}:\ ${project.org.sleuthkit.autopsy.thunderbirdparser}:\ - ${project.org.sleuthkit.autopsy.exifparser}:\ ${project.org.sleuthkit.autopsy.core}:\ ${project.org.sleuthkit.autopsy.corelibs}:\ - ${project.org.sleuthkit.autopsy.sevenzip}:\ ${project.org.sleuthkit.autopsy.scalpel}:\ - ${project.org.sleuthkit.autopsy.timeline}:\ - ${project.org.sleuthkit.autopsy.filetypeid}:\ - ${project.org.sleuthkit.autopsy.ewfverify}:\ - ${project.org.sleuthkit.autopsy.fileextmismatch} + ${project.org.sleuthkit.autopsy.ewfverify} project.org.sleuthkit.autopsy.core=Core project.org.sleuthkit.autopsy.corelibs=CoreLibs project.org.sleuthkit.autopsy.hashdatabase=HashDatabase @@ -44,9 +38,5 @@ project.org.sleuthkit.autopsy.keywordsearch=KeywordSearch project.org.sleuthkit.autopsy.recentactivity=RecentActivity project.org.sleuthkit.autopsy.testing=Testing project.org.sleuthkit.autopsy.thunderbirdparser=thunderbirdparser -project.org.sleuthkit.autopsy.exifparser=ExifParser -project.org.sleuthkit.autopsy.sevenzip=SevenZip project.org.sleuthkit.autopsy.scalpel=ScalpelCarver -project.org.sleuthkit.autopsy.timeline=Timeline -project.org.sleuthkit.autopsy.filetypeid=FileTypeId diff --git a/test/script/regression.py b/test/script/regression.py index 0d09f72a22..7c33e62d33 100755 --- a/test/script/regression.py +++ b/test/script/regression.py @@ -34,16 +34,13 @@ import xml from time import localtime, strftime from xml.dom.minidom import parse, parseString import smtplib -from email.mime.image import MIMEImage -from email.mime.multipart import MIMEMultipart -from email.mime.text import MIMEText import re import zipfile import zlib -import Emailer import srcupdater from regression_utils import * - +import shutil +import ntpath # # Please read me... # @@ -130,7 +127,6 @@ class TestRunner(object): logres =[] for test_data in test_data_list: Errors.clear_print_logs() - Errors.set_testing_phase(test_data.image) if not (test_config.args.rebuild or os.path.exists(test_data.gold_archive)): msg = "Gold standard doesn't exist, skipping image:" Errors.print_error(msg) @@ -148,28 +144,16 @@ class TestRunner(object): time.sleep(10) Reports.write_html_foot(test_config.html_log) - # TODO: move this elsewhere - if (len(logres)>0): - for lm in logres: - for ln in lm: - Errors.add_email_msg(ln) - # TODO: possibly worth putting this in a sub method if all([ test_data.overall_passed for test_data in test_data_list ]): - Errors.add_email_msg("All images passed.\n") + pass else: - msg = "The following images failed:\n" - for test_data in test_data_list: - if not test_data.overall_passed: - msg += "\t" + test_data.image + "\n" - Errors.add_email_msg(msg) html = open(test_config.html_log) - Errors.add_email_attachment(html.name) + Errors.add_errors_out(html.name) html.close() - - if test_config.email_enabled: - Emailer.send_email(test_config.mail_to, test_config.mail_server, - test_config.mail_subject, Errors.email_body, Errors.email_attachs) + + if test_config.jenkins: + setupAttachments(Errors.errors_out, test_config) def _run_autopsy_ingest(test_data): """Run Autopsy ingest for the image in the given TestData. @@ -231,7 +215,11 @@ class TestRunner(object): Reports.generate_reports(test_data) if(not test_data.overall_passed): - Errors.add_email_attachment(test_data.common_log_path) + diffFiles = [ f for f in os.listdir(test_data.output_path) if os.path.isfile(os.path.join(test_data.output_path,f)) ] + for f in diffFiles: + if f.endswith("Diff.txt"): + Errors.add_errors_out(os.path.join(test_data.output_path, f)) + Errors.add_errors_out(test_data.common_log_path) return logres def _extract_gold(test_data): @@ -299,7 +287,6 @@ class TestRunner(object): shutil.copy(test_data.sorted_log, error_pth) except IOError as e: Errors.print_error(str(e)) - Errors.add_email_message("Not rebuilt properly") print(str(e)) print(traceback.format_exc()) # Rebuild the HTML report @@ -361,6 +348,8 @@ class TestRunner(object): test_data.ant.append("-Dgold_path=" + test_config.gold) test_data.ant.append("-Dout_path=" + make_local_path(test_data.output_path)) + if test_config.jenkins: + test_data.ant.append("-Ddiff_dir="+ test_config.diff_dir) test_data.ant.append("-Dignore_unalloc=" + "%s" % test_config.args.unallocated) test_data.ant.append("-Dtest.timeout=" + str(test_config.timeout)) @@ -578,6 +567,7 @@ class TestConfiguration(object): images: a listof_Image, the images to be tested timeout: a Nat, the amount of time before killing the test ant: a listof_String, the ant command to run the tests + jenkins: a boolean, is this test running through a Jenkins job? """ def __init__(self, args): @@ -604,11 +594,7 @@ class TestConfiguration(object): # Infinite Testing info timer = 0 self.images = [] - # Email info - self.email_enabled = args.email_enabled - self.mail_server = "" - self.mail_to = "" - self.mail_subject = "" + self.jenkins = False # Set the timeout to something huge # The entire tester should not timeout before this number in ms # However it only seems to take about half this time @@ -646,15 +632,18 @@ class TestConfiguration(object): if parsed_config.getElementsByTagName("golddir"): self.gold = parsed_config.getElementsByTagName("golddir")[0].getAttribute("value").encode().decode("utf_8") self.img_gold = make_path(self.gold, 'tmp') - + if parsed_config.getElementsByTagName("jenkins"): + self.jenkins = True + if parsed_config.getElementsByTagName("diffdir"): + self.diff_dir = parsed_config.getElementsByTagName("diffdir")[0].getAttribute("value").encode().decode("utf_8") + else: + self.jenkins = False self._init_imgs(parsed_config) self._init_build_info(parsed_config) - self._init_email_info(parsed_config) except IOError as e: msg = "There was an error loading the configuration file.\n" msg += "\t" + str(e) - Errors.add_email_msg(msg) logging.critical(traceback.format_exc()) print(traceback.format_exc()) @@ -687,7 +676,6 @@ class TestConfiguration(object): else: msg = "File: " + value + " doesn't exist" Errors.print_error(msg) - Errors.add_email_msg(msg) image_count = len(self.images) # Sanity check to see if there are obvious gold images that we are not testing @@ -701,27 +689,6 @@ class TestConfiguration(object): elif (image_count < gold_count): print("******Alert: There are more gold standards than input images, this will not check all gold Standards.\n") - def _init_email_info(self, parsed_config): - """Initializes email information dictionary""" - email_elements = parsed_config.getElementsByTagName("email") - if email_elements: - mail_to = email_elements[0] - self.mail_to = mail_to.getAttribute("value").encode().decode("utf_8") - mail_server_elements = parsed_config.getElementsByTagName("mail_server") - if mail_server_elements: - mail_from = mail_server_elements[0] - self.mail_server = mail_from.getAttribute("value").encode().decode("utf_8") - subject_elements = parsed_config.getElementsByTagName("subject") - if subject_elements: - subject = subject_elements[0] - self.mail_subject = subject.getAttribute("value").encode().decode("utf_8") - if self.mail_server and self.mail_to and self.args.email_enabled: - self.email_enabled = True - print("Email will be sent to ", self.mail_to) - else: - self.email_enabled = False - print("No email will be sent.") - #-------------------------------------------------# # Functions relating to comparing outputs # @@ -799,11 +766,7 @@ class TestResultsDiffer(object): diff_file = codecs.open(diff_path, "wb", "utf_8") dffcmdlst = ["diff", output_file, gold_file] subprocess.call(dffcmdlst, stdout = diff_file) - Errors.add_email_attachment(diff_path) - msg = "There was a difference in " - msg += os.path.basename(output_file) + ".\n" - Errors.add_email_msg(msg) - Errors.print_error(msg) + Errors.add_errors_out(diff_path) return False else: return True @@ -1242,7 +1205,6 @@ class Logs(object): # Set the TestData start time based off the first line of autopsy.log.0 # *** If logging time format ever changes this will break *** test_data.start_date = log.readline().split(" org.")[0] - # Set the test_data ending time based off the "create" time (when the file was copied) test_data.end_date = time.ctime(os.path.getmtime(log_path)) except IOError as e: @@ -1261,27 +1223,27 @@ class Logs(object): version_line = search_logs("INFO: Application name: Autopsy, version:", test_data)[0] test_data.autopsy_version = get_word_at(version_line, 5).rstrip(",") - test_data.heap_space = search_logs("Heap memory usage:", test_data)[0].rstrip().split(": ")[1] - + ingest_line = search_logs("Ingest (including enqueue)", test_data)[0] test_data.total_ingest_time = get_word_at(ingest_line, 6).rstrip() - - message_line = search_log_set("autopsy", "Ingest messages count:", test_data)[0] - test_data.ingest_messages = int(message_line.rstrip().split(": ")[2]) - - files_line = search_log_set("autopsy", "Indexed files count:", test_data)[0] - test_data.indexed_files = int(files_line.rstrip().split(": ")[2]) - - chunks_line = search_log_set("autopsy", "Indexed file chunks count:", test_data)[0] - test_data.indexed_chunks = int(chunks_line.rstrip().split(": ")[2]) + + message_line_count = find_msg_in_log_set("Ingest messages count:", test_data) + test_data.indexed_files = message_line_count + + files_line_count = find_msg_in_log_set("Indexed files count:", test_data) + test_data.indexed_files = files_line_count + + chunks_line_count = find_msg_in_log_set("Indexed file chunks count:", test_data) + test_data.indexed_chunks = chunks_line_count + except (OSError, IOError) as e: Errors.print_error("Error: Unable to find the required information to fill test_config data.") Errors.print_error(str(e) + "\n") logging.critical(traceback.format_exc()) print(traceback.format_exc()) try: - service_lines = search_log("autopsy.log.0", "to process()", test_data) + service_lines = find_msg_in_log("autopsy.log.0", "to process()", test_data) service_list = [] for line in service_lines: words = line.split(" ") @@ -1464,24 +1426,12 @@ class Errors: Attributes: printout: a listof_String, the non-error messages that were printed printerror: a listof_String, the error messages that were printed - email_body: a String, the body of the report email - email_msg_prefix: a String, the prefix for lines added to the email email_attchs: a listof_pathto_File, the files to be attached to the report email """ printout = [] printerror = [] - email_body = "" - email_msg_prefix = "Configuration" - email_attachs = [] - - def set_testing_phase(image_name): - """Change the email message prefix to be the given testing phase. - - Args: - image_name: a String, representing the current image being tested - """ - Errors.email_msg_prefix = image_name + errors_out = [] def print_out(msg): """Print out an informational message. @@ -1506,21 +1456,13 @@ class Errors: Errors.printout = [] Errors.printerror = [] - def add_email_msg(msg): - """Add the given message to the body of the report email. - - Args: - msg: a String, the message to be added to the email - """ - Errors.email_body += Errors.email_msg_prefix + ":" + msg - - def add_email_attachment(path): + def add_errors_out(path): """Add the given file to be an attachment for the report email Args: file: a pathto_File, the file to add """ - Errors.email_attachs.append(path) + Errors.errors_out.append(path) class DiffResults(object): @@ -1602,7 +1544,6 @@ class Args(object): self.exception = False self.exception_string = "" self.fr = False - self.email_enabled = False def parse(self): """Get the command line arguments and parse them.""" @@ -1662,8 +1603,6 @@ class Args(object): elif arg == "-fr" or arg == "--forcerun": print("Not downloading new images") self.fr = True - elif arg == "--email": - self.email_enabled = True else: print(usage()) return False @@ -1769,6 +1708,40 @@ def search_log_set(type, string, test_data): log.close() return results +def find_msg_in_log_set(string, test_data): + """Count how many strings of a certain type are in a log set. + + Args: + string: the String to search for. + test_data: the TestData containing the logs to search. + Returns: + an int, the number of occurances of the string type. + """ + count = 0 + try: + line = search_log_set("autopsy", string, test_data)[0] + count = int(line.rstrip().split(": ")[2]) + except (Exception) as e: + # there weren't any matching messages found + pass + return count + +def find_msg_in_log(log, string, test_data): + """Get the strings of a certain type that are in a log. + + Args: + string: the String to search for. + test_data: the TestData containing the log to search. + Returns: + a listof_String, the lines on which the String was found. + """ + lines = [] + try: + lines = search_log("autopsy.log.0", string, test_data)[0] + except (Exception) as e: + # there weren't any matching messages found + pass + return lines def clear_dir(dir): """Clears all files from a directory and remakes it. @@ -1838,11 +1811,34 @@ def find_file_in_dir(dir, name, ext): except: raise DirNotFoundException(dir) +def setupAttachments(attachments, test_config): + """Move email attachments to the location specified in the config file. + Used for Jenkins build. + + Args: + attachments: a listof_String, the files to be moved + test_config: TestConfiguration, used to determine where to move the files to + """ + call = ['pwd'] + subprocess.call(call) + + # remove old diff files + filelist = [f for f in os.listdir(test_config.diff_dir) if (f.endswith(".txt") or f.endswith(".html"))] + for f in filelist: + if os.path.isfile(test_config.diff_dir + "/" + f): + os.remove(test_config.diff_dir + "/" + f) + + # move in the new diff files + for file in attachments: + filename = ntpath.basename(file) + destination = os.path.join(test_config.diff_dir, filename) + call = ['cp', file, destination] + subprocess.call(call) + class OS: LINUX, MAC, WIN, CYGWIN = range(4) - if __name__ == "__main__": global SYS if _platform == "linux" or _platform == "linux2": diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java index d2d332f439..289f1fa5a0 100644 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java @@ -54,7 +54,6 @@ public final class ThunderbirdMboxFileIngestModule extends IngestModuleAdapter i private static final Logger logger = Logger.getLogger(ThunderbirdMboxFileIngestModule.class.getName()); private IngestServices services = IngestServices.getInstance(); - private int messageId = 0; // RJCTODO: Not thread safe private FileManager fileManager; private IngestJobContext context; @@ -122,7 +121,7 @@ public final class ThunderbirdMboxFileIngestModule extends IngestModuleAdapter i if (abstractFile.getSize() >= services.getFreeDiskSpace()) { logger.log(Level.WARNING, "Not enough disk space to write file to disk."); - IngestMessage msg = IngestMessage.createErrorMessage(messageId++, EmailParserModuleFactory.getModuleName(), EmailParserModuleFactory.getModuleName(), + IngestMessage msg = IngestMessage.createErrorMessage(EmailParserModuleFactory.getModuleName(), EmailParserModuleFactory.getModuleName(), NbBundle.getMessage(this.getClass(), "ThunderbirdMboxFileIngestModule.processPst.errMsg.outOfDiskSpace", abstractFile.getName())); @@ -397,7 +396,7 @@ public final class ThunderbirdMboxFileIngestModule extends IngestModuleAdapter i } void postErrorMessage(String subj, String details) { - IngestMessage ingestMessage = IngestMessage.createErrorMessage(messageId++, EmailParserModuleFactory.getModuleVersion(), subj, details); + IngestMessage ingestMessage = IngestMessage.createErrorMessage(EmailParserModuleFactory.getModuleVersion(), subj, details); services.postMessage(ingestMessage); }