Merge remote-tracking branch 'upstream/develop' into merge-develop-8-14-15

Core/src/org/sleuthkit/autopsy/datamodel/FileTypeExtensions.java

@@ -37,7 +37,7 @@ public class FileTypeExtensions {
     private final static List<String> TEXT_EXTENSIONS = Arrays.asList(".txt", ".rtf", ".log", ".text", ".xml"); //NON-NLS
     private final static List<String> WEB_EXTENSIONS = Arrays.asList(".html", ".htm", ".css", ".js", ".php", ".aspx"); //NON-NLS
     private final static List<String> PDF_EXTENSIONS = Arrays.asList(".pdf"); //NON-NLS
-    private final static List<String> ARCHIVE_EXTENSIONS = Arrays.asList(".zip", ".rar", ".7zip", ".7z", ".arj", ".tar", ".gzip", ".bzip", ".bzip2", ".cab", ".jar", ".cpio", ".ar", ".gz", ".tgz", ".bz2"); //NON-NLS
+    private final static List<String> ARCHIVE_EXTENSIONS = Arrays.asList(".zip", ".rar", ".7zip", ".7z", ".arj", ".tar", ".gzip", ".bzip", ".bzip2", ".cab", ".jar", ".cpio", ".ar", ".gz", ".tgz"); //NON-NLS
 
     public static List<String> getImageExtensions() {
         return IMAGE_EXTENSIONS;

Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/Bundle.properties

@@ -37,4 +37,3 @@ EmbeddedFileExtractorIngestModule.ImageExtractor.xlsContainer.init.err=Xls conta
 EmbeddedFileExtractorIngestModule.ImageExtractor.xlsxContainer.init.err=Xlsx container could not be initialized while reading: {0}
 EmbeddedFileExtractorIngestModule.ImageExtractor.extractImage.addToDB.exception.msg=Unable to add the derived files to the database.
 EmbeddedFileExtractorIngestModule.ImageExtractor.getOutputFolderPath.exception.msg=Could not get path for image extraction from Abstract File: {0}
-EmbeddedFileExtractorIngestModule.ArchiveExtractor.UnpackStream.write.noSpace.msg=Unable to write content to disk. Not enough space.

Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java

@@ -24,8 +24,6 @@ import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.OutputStream;
-import java.nio.file.Files;
-import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Date;
@@ -42,6 +40,7 @@ import net.sf.sevenzipjbinding.simple.ISimpleInArchive;
 import net.sf.sevenzipjbinding.simple.ISimpleInArchiveItem;
 import org.netbeans.api.progress.ProgressHandle;
 import org.netbeans.api.progress.ProgressHandleFactory;
+import org.openide.util.Exceptions;
 import org.openide.util.NbBundle;
 import org.sleuthkit.autopsy.casemodule.Case;
 import org.sleuthkit.autopsy.casemodule.services.FileManager;
@@ -99,8 +98,7 @@ class SevenZipExtractor {
         GZIP("application/gzip"),
         XGZIP("application/x-gzip"),
         XBZIP2("application/x-bzip2"),
-        XTAR("application/x-tar"),
+        XTAR("application/x-tar");
-        XGTAR("application/x-gtar");
 
         private final String mimeType;
 
@@ -123,9 +121,9 @@ class SevenZipExtractor {
                 logger.log(Level.INFO, "7-Zip-JBinding library was initialized on supported platform: {0}", platform); //NON-NLS
             } catch (SevenZipNativeInitializationException e) {
                 logger.log(Level.SEVERE, "Error initializing 7-Zip-JBinding library", e); //NON-NLS
-                String msg = NbBundle.getMessage(SevenZipExtractor.class, "EmbeddedFileExtractorIngestModule.ArchiveExtractor.init.errInitModule.msg",
+                String msg = NbBundle.getMessage(this.getClass(), "EmbeddedFileExtractorIngestModule.ArchiveExtractor.init.errInitModule.msg",
                         EmbeddedFileExtractorModuleFactory.getModuleName());
-                String details = NbBundle.getMessage(SevenZipExtractor.class, "EmbeddedFileExtractorIngestModule.ArchiveExtractor.init.errCantInitLib",
+                String details = NbBundle.getMessage(this.getClass(), "EmbeddedFileExtractorIngestModule.ArchiveExtractor.init.errCantInitLib",
                         e.getMessage());
                 services.postMessage(IngestMessage.createErrorMessage(EmbeddedFileExtractorModuleFactory.getModuleName(), msg, details));
                 throw new IngestModuleException(e.getMessage());
@@ -206,7 +204,7 @@ class SevenZipExtractor {
             if (cRatio >= MAX_COMPRESSION_RATIO) {
                 String itemName = archiveFileItem.getPath();
                 logger.log(Level.INFO, "Possible zip bomb detected, compression ration: {0} for in archive item: {1}", new Object[]{cRatio, itemName}); //NON-NLS
-                String msg = NbBundle.getMessage(SevenZipExtractor.class,
+                String msg = NbBundle.getMessage(this.getClass(),
                         "EmbeddedFileExtractorIngestModule.ArchiveExtractor.isZipBombCheck.warnMsg", archiveFile.getName(), itemName);
                 String path;
                 try {
@@ -214,7 +212,7 @@ class SevenZipExtractor {
                 } catch (TskCoreException ex) {
                     path = archiveFile.getParentPath() + archiveFile.getName();
                 }
-                String details = NbBundle.getMessage(SevenZipExtractor.class,
+                String details = NbBundle.getMessage(this.getClass(),
                         "EmbeddedFileExtractorIngestModule.ArchiveExtractor.isZipBombCheck.warnDetails", cRatio, path);
                 //MessageNotifyUtil.Notify.error(msg, details);
                 services.postMessage(IngestMessage.createWarningMessage(EmbeddedFileExtractorModuleFactory.getModuleName(), msg, details));
@@ -313,9 +311,9 @@ class SevenZipExtractor {
         if (parentAr == null) {
             parentAr = archiveDepthCountTree.addArchive(null, archiveId);
         } else if (parentAr.getDepth() == MAX_DEPTH) {
-            String msg = NbBundle.getMessage(SevenZipExtractor.class,
+            String msg = NbBundle.getMessage(this.getClass(),
                     "EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.warnMsg.zipBomb", archiveFile.getName());
-            String details = NbBundle.getMessage(SevenZipExtractor.class,
+            String details = NbBundle.getMessage(this.getClass(),
                     "EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.warnDetails.zipBomb",
                     parentAr.getDepth(), archiveFilePath);
             //MessageNotifyUtil.Notify.error(msg, details);
@@ -330,7 +328,7 @@ class SevenZipExtractor {
         SevenZipContentReadStream stream = null;
 
         final ProgressHandle progress = ProgressHandleFactory.createHandle(
-                NbBundle.getMessage(SevenZipExtractor.class, "EmbeddedFileExtractorIngestModule.ArchiveExtractor.moduleName"));
+                NbBundle.getMessage(this.getClass(), "EmbeddedFileExtractorIngestModule.ArchiveExtractor.moduleName"));
         int processedItems = 0;
 
         boolean progressStarted = false;
@@ -402,7 +400,7 @@ class SevenZipExtractor {
                         pathInArchive = "/" + useName;
                     }
 
-                    String msg = NbBundle.getMessage(SevenZipExtractor.class, "EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.unknownPath.msg",
+                    String msg = NbBundle.getMessage(this.getClass(), "EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.unknownPath.msg",
                             archiveFilePath, pathInArchive);
                     logger.log(Level.WARNING, msg);
 
@@ -434,19 +432,24 @@ class SevenZipExtractor {
                     fullEncryption = false;
                 }
 
-                // NOTE: item.getSize() may return null in case of certain
+                final Long size = item.getSize();
-                // archiving formats. Eg: BZ2
+                if (size == null) {
-                Long size = item.getSize();
+                    // If the size property cannot be determined, out-of-disk-space
+                    // situations cannot be ascertained.
+                    // Hence skip this file.
+                    logger.log(Level.WARNING, "Size cannot be determined. Skipping file in archive: {0}", pathInArchive); //NON-NLS
+                    continue;
+                }
 
                 //check if unpacking this file will result in out of disk space
                 //this is additional to zip bomb prevention mechanism
-                if (freeDiskSpace != IngestMonitor.DISK_FREE_SPACE_UNKNOWN && size != null && size > 0) { //if free space is known and file is not empty.
+                if (freeDiskSpace != IngestMonitor.DISK_FREE_SPACE_UNKNOWN && size > 0) { //if known free space and file not empty
                     long newDiskSpace = freeDiskSpace - size;
                     if (newDiskSpace < MIN_FREE_DISK_SPACE) {
-                        String msg = NbBundle.getMessage(SevenZipExtractor.class,
+                        String msg = NbBundle.getMessage(this.getClass(),
                                 "EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.notEnoughDiskSpace.msg",
                                 archiveFilePath, fileName);
-                        String details = NbBundle.getMessage(SevenZipExtractor.class,
+                        String details = NbBundle.getMessage(this.getClass(),
                                 "EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.notEnoughDiskSpace.details");
                         //MessageNotifyUtil.Notify.error(msg, details);
                         services.postMessage(IngestMessage.createErrorMessage(EmbeddedFileExtractorModuleFactory.getModuleName(), msg, details));
@@ -498,20 +501,21 @@ class SevenZipExtractor {
                 final long modtime = writeTime == null ? 0L : writeTime.getTime() / 1000;
                 final long accesstime = accessTime == null ? 0L : accessTime.getTime() / 1000;
 
+                //record derived data in unode, to be traversed later after unpacking the archive
+                unpackedNode.addDerivedInfo(size, !isDir,
+                        0L, createtime, accesstime, modtime, localRelPath);
+
                 //unpack locally if a file
-                SevenZipExtractor.UnpackStream unpackStream = null;
                 if (!isDir) {
+                    SevenZipExtractor.UnpackStream unpackStream = null;
                     try {
-                        unpackStream = new SevenZipExtractor.UnpackStream(localAbsPath, freeDiskSpace, size == null);
+                        unpackStream = new SevenZipExtractor.UnpackStream(localAbsPath);
                         item.extractSlow(unpackStream);
                     } catch (Exception e) {
                         //could be something unexpected with this file, move on
                         logger.log(Level.WARNING, "Could not extract file from archive: " + localAbsPath, e); //NON-NLS
                     } finally {
                         if (unpackStream != null) {
-                            //record derived data in unode, to be traversed later after unpacking the archive
-                            unpackedNode.addDerivedInfo(unpackStream.getNumberOfBytesWritten(), !isDir,
-                                    0L, createtime, accesstime, modtime, localRelPath);
                             unpackStream.close();
                         }
                     }
@@ -545,9 +549,9 @@ class SevenZipExtractor {
 
             // print a message if the file is allocated
             if (archiveFile.isMetaFlagSet(TskData.TSK_FS_META_FLAG_ENUM.ALLOC)) {
-                String msg = NbBundle.getMessage(SevenZipExtractor.class, "EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.errUnpacking.msg",
+                String msg = NbBundle.getMessage(this.getClass(), "EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.errUnpacking.msg",
                         archiveFile.getName());
-                String details = NbBundle.getMessage(SevenZipExtractor.class,
+                String details = NbBundle.getMessage(this.getClass(),
                         "EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.errUnpacking.details",
                         archiveFilePath, ex.getMessage());
                 services.postMessage(IngestMessage.createErrorMessage(EmbeddedFileExtractorModuleFactory.getModuleName(), msg, details));
@@ -586,8 +590,8 @@ class SevenZipExtractor {
                 logger.log(Level.SEVERE, "Error creating blackboard artifact for encryption detected for file: " + archiveFilePath, ex); //NON-NLS
             }
 
-            String msg = NbBundle.getMessage(SevenZipExtractor.class, "EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.encrFileDetected.msg");
+            String msg = NbBundle.getMessage(this.getClass(), "EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.encrFileDetected.msg");
-            String details = NbBundle.getMessage(SevenZipExtractor.class,
+            String details = NbBundle.getMessage(this.getClass(),
                     "EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.encrFileDetected.details",
                     archiveFile.getName(), EmbeddedFileExtractorModuleFactory.getModuleName());
             services.postMessage(IngestMessage.createWarningMessage(EmbeddedFileExtractorModuleFactory.getModuleName(), msg, details));
@@ -608,15 +612,8 @@ class SevenZipExtractor {
 
         private OutputStream output;
         private String localAbsPath;
-        private long freeDiskSpace;
-        private boolean sizeUnknown = false;
-        private boolean outOfSpace = false;
-        private long bytesWritten = 0;
 
-        UnpackStream(String localAbsPath, long freeDiskSpace, boolean sizeUnknown) {
+        UnpackStream(String localAbsPath) {
-            this.sizeUnknown = sizeUnknown;
-            this.freeDiskSpace = freeDiskSpace;
-            this.localAbsPath = localAbsPath;
             try {
                 output = new BufferedOutputStream(new FileOutputStream(localAbsPath));
             } catch (FileNotFoundException ex) {
@@ -625,38 +622,13 @@ class SevenZipExtractor {
 
         }
 
-        public long getNumberOfBytesWritten() {
-            return this.bytesWritten;
-        }
-
         @Override
         public int write(byte[] bytes) throws SevenZipException {
             try {
-                if (!sizeUnknown) {
                 output.write(bytes);
-                } else {
-                    // If the content size is unknown, cautiously write to disk.
-                    // Write only if byte array is less than 80% of the current
-                    // free disk space.
-                    if (freeDiskSpace == IngestMonitor.DISK_FREE_SPACE_UNKNOWN || bytes.length < 0.8 * freeDiskSpace) {
-                        output.write(bytes);
-                        // NOTE: this method is called multiple times for a
-                        // single extractSlow() call. Update bytesWritten and
-                        // freeDiskSpace after every write operation.
-                        this.bytesWritten += bytes.length;
-                        this.freeDiskSpace -= bytes.length;
-                    } else {
-                        this.outOfSpace = true;
-                        logger.log(Level.INFO, NbBundle.getMessage(
-                                SevenZipExtractor.class,
-                                "EmbeddedFileExtractorIngestModule.ArchiveExtractor.UnpackStream.write.noSpace.msg"));
-                        throw new SevenZipException(
-                                NbBundle.getMessage(SevenZipExtractor.class, "EmbeddedFileExtractorIngestModule.ArchiveExtractor.UnpackStream.write.noSpace.msg"));
-                    }
-                }
             } catch (IOException ex) {
                 throw new SevenZipException(
-                        NbBundle.getMessage(SevenZipExtractor.class, "EmbeddedFileExtractorIngestModule.ArchiveExtractor.UnpackStream.write.exception.msg",
+                        NbBundle.getMessage(this.getClass(), "EmbeddedFileExtractorIngestModule.ArchiveExtractor.UnpackStream.write.exception.msg",
                                 localAbsPath), ex);
             }
             return bytes.length;
@@ -667,9 +639,6 @@ class SevenZipExtractor {
                 try {
                     output.flush();
                     output.close();
-                    if (this.outOfSpace) {
-                        Files.delete(Paths.get(this.localAbsPath));
-                    }
                 } catch (IOException e) {
                     logger.log(Level.SEVERE, "Error closing unpack stream for file: {0}", localAbsPath); //NON-NLS
                 }
@@ -805,7 +774,7 @@ class SevenZipExtractor {
             } catch (TskCoreException ex) {
                 logger.log(Level.SEVERE, "Error adding a derived file to db:" + fileName, ex); //NON-NLS
                 throw new TskCoreException(
-                        NbBundle.getMessage(SevenZipExtractor.class, "EmbeddedFileExtractorIngestModule.ArchiveExtractor.UnpackedTree.exception.msg",
+                        NbBundle.getMessage(this.getClass(), "EmbeddedFileExtractorIngestModule.ArchiveExtractor.UnpackedTree.exception.msg",
                                 fileName), ex);
             }
 

Core/src/org/sleuthkit/autopsy/modules/exif/ExifParserFileIngestModule.java

@@ -32,6 +32,7 @@ import java.io.InputStream;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Date;
+import java.util.HashSet;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.logging.Level;
 import org.openide.util.NbBundle;
@@ -65,8 +66,12 @@ public final class ExifParserFileIngestModule implements FileIngestModule {
     private long jobId;
     private static final IngestModuleReferenceCounter refCounter = new IngestModuleReferenceCounter();
     private FileTypeDetector fileTypeDetector;
+    private final HashSet<String> supportedMimeTypes = new HashSet<>();
 
     ExifParserFileIngestModule() {
+        supportedMimeTypes.add("audio/x-wav");
+        supportedMimeTypes.add("image/jpeg");
+        supportedMimeTypes.add("image/tiff");
     }
 
     @Override
@@ -206,7 +211,7 @@ public final class ExifParserFileIngestModule implements FileIngestModule {
         try {
             String mimeType = fileTypeDetector.getFileType(f);
             if (mimeType != null) {
-                return fileTypeDetector.getFileType(f).equals("image/jpeg");
+                return supportedMimeTypes.contains(mimeType);
             } else {
                 return false;
             }

Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeDetector.java

@@ -167,7 +167,8 @@ public class FileTypeDetector {
         // as octet-stream.
         if (!file.isFile() || file.getSize() <= 0
                 || (file.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS)
-                || (file.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.UNUSED_BLOCKS)) {
+                || (file.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.UNUSED_BLOCKS)
+                || (file.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.VIRTUAL_DIR)) {
             return MimeTypes.OCTET_STREAM;
         }
 

Core/src/org/sleuthkit/autopsy/report/ReportHTML.java

@@ -562,13 +562,6 @@ class ReportHTML implements TableReportModule {
             return;
         }
         AbstractFile file = (AbstractFile) content;
-        // Don't make a local copy of the file if it is a directory or unallocated space.
-        if (file.isDir()
-                || file.getType() == TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS
-                || file.getType() == TSK_DB_FILES_TYPE_ENUM.UNUSED_BLOCKS) {
-            row.add("");
-            return;
-        }
 
         // Add metadata about the file to HTML output
         row.add(file.getMtimeAsDate());
@@ -578,14 +571,18 @@ class ReportHTML implements TableReportModule {
         row.add(Long.toString(file.getSize()));
         row.add(file.getMd5Hash());
 
-        // save it in a folder based on the tag name
-        String localFilePath = saveContent(file, contentTag.getName().getDisplayName());
-
         // Add the hyperlink to the row. A column header for it was created in startTable().
         StringBuilder localFileLink = new StringBuilder();
+        // Don't make a local copy of the file if it is a directory or unallocated space.
+        if (!(file.isDir()
+                || file.getType() == TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS
+                || file.getType() == TSK_DB_FILES_TYPE_ENUM.UNUSED_BLOCKS)) {
             localFileLink.append("<a href=\""); //NON-NLS
+            // save it in a folder based on the tag name
+            String localFilePath = saveContent(file, contentTag.getName().getDisplayName());
             localFileLink.append(localFilePath);
             localFileLink.append("\">");
+        }
 
         StringBuilder builder = new StringBuilder();
         builder.append("\t<tr>\n"); //NON-NLS

pythonExamples/Aug2015DataSourceTutorial/FindContactsDb.py

@@ -135,16 +135,16 @@ class ContactsDbIngestModule(DataSourceIngestModule):
             try: 
                 Class.forName("org.sqlite.JDBC").newInstance()
                 dbConn = DriverManager.getConnection("jdbc:sqlite:%s"  % lclDbPath)
-            except:
+            except SQLException as e:
-                self.log(Level.INFO, "Could not open database file (not SQLite) " + file.getName())
+                self.log(Level.INFO, "Could not open database file (not SQLite) " + file.getName() + " (" + e.getMessage() + ")")
                 return IngestModule.ProcessResult.OK
             
             # Query the contacts table in the database and get all columns. 
             try:
                 stmt = dbConn.createStatement()
                 resultSet = stmt.executeQuery("SELECT * FROM contacts")
-            except:
+            except SQLException as e:
-                self.log(Level.INFO, "Error querying database for contacts table")
+                self.log(Level.INFO, "Error querying database for contacts table (" + e.getMessage() + ")")
                 return IngestModule.ProcessResult.OK
 
             # Cycle through each row and create artifacts
@@ -153,8 +153,8 @@ class ContactsDbIngestModule(DataSourceIngestModule):
                     name  = resultSet.getString("name")
                     email = resultSet.getString("email")
                     phone = resultSet.getString("phone")
-                except:
+                except SQLException as e:
-                    self.log(Level.INFO, "Error getting values from contacts table")
+                    self.log(Level.INFO, "Error getting values from contacts table (" + e.getMessage() + ")")
                 
                 
                 # Make an artifact on the blackboard, TSK_CONTACT and give it attributes for each of the fields

pythonExamples/Aug2015DataSourceTutorial/RunExe.py

@@ -134,6 +134,8 @@ class RunExeIngestModule(DataSourceIngestModule):
         reportHandle = open(reportPath, 'w')
         
         # Run the EXE, saving output to the report
+        # NOTE: we should really be checking for if the module has been
+        # cancelled and then killing the process. 
         self.log(Level.INFO, "Running program on data source")
         subprocess.Popen([self.path_to_exe, imagePaths[0]], stdout=reportHandle).communicate()[0]    
         reportHandle.close()