Merge pull request #7069 from sleuthkit/release-4.19.0

Merge release 4.19.0 into develop

Core/src/org/sleuthkit/autopsy/casemodule/events/OsAcctInstancesAddedEvent.java

@@ -18,7 +18,6 @@
  */
 package org.sleuthkit.autopsy.casemodule.events;
 
-import java.util.ArrayList;
 import java.util.List;
 import static org.sleuthkit.autopsy.casemodule.Case.Events.OS_ACCT_INSTANCES_ADDED;
 import org.sleuthkit.datamodel.OsAccountInstance;
@@ -26,16 +25,16 @@ import org.sleuthkit.datamodel.SleuthkitCase;
 import org.sleuthkit.datamodel.TskCoreException;
 
 /**
- * An application event published when OS accounts are added to the Sleuth Kit
+ * An application event published when OS account instances are added to the
- * data model for a case.
+ * Sleuth Kit data model for a case.
  */
 public final class OsAcctInstancesAddedEvent extends TskDataModelChangedEvent<OsAccountInstance, OsAccountInstance> {
 
     private static final long serialVersionUID = 1L;
 
     /**
-     * Constructs an application event published when OS account instances are added to
+     * Constructs an application event published when OS account instances are
-     * the Sleuth Kit data model for a case.
+     * added to the Sleuth Kit data model for a case.
      *
      * @param osAcctInstances The OS account instances that were added.
      */
@@ -44,9 +43,9 @@ public final class OsAcctInstancesAddedEvent extends TskDataModelChangedEvent<Os
     }
 
     /**
-     * Gets the OS accounts that have been added or updated.
+     * Gets the OS account instances that have been added.
      *
-     * @return The OS accounts.
+     * @return The OS account instances.
      */
     public List<OsAccountInstance> getOsAccountInstances() {
         return getNewValue();
@@ -54,11 +53,7 @@ public final class OsAcctInstancesAddedEvent extends TskDataModelChangedEvent<Os
 
     @Override
     protected List<OsAccountInstance> getNewValueObjects(SleuthkitCase caseDb, List<Long> ids) throws TskCoreException {
-        List<OsAccountInstance> osAccountInstances = new ArrayList<>();
+        return caseDb.getOsAccountManager().getOsAccountInstances(ids);
-        for (Long id : ids) {
-            //RJCTODO
-        }
-        return osAccountInstances;
     }
 
-}
+}

Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/CaseEventListener.java

@@ -37,6 +37,7 @@ import org.sleuthkit.autopsy.casemodule.events.ContentTagAddedEvent;
 import org.sleuthkit.autopsy.casemodule.events.ContentTagDeletedEvent;
 import org.sleuthkit.autopsy.casemodule.events.DataSourceAddedEvent;
 import org.sleuthkit.autopsy.casemodule.events.DataSourceNameChangedEvent;
+import org.sleuthkit.autopsy.casemodule.events.OsAcctInstancesAddedEvent;
 import org.sleuthkit.autopsy.casemodule.services.TagsManager;
 import org.sleuthkit.autopsy.coreutils.Logger;
 import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeInstance;
@@ -56,6 +57,7 @@ import org.sleuthkit.datamodel.TskData;
 import org.sleuthkit.autopsy.centralrepository.datamodel.CentralRepository;
 import org.sleuthkit.datamodel.Tag;
 import org.sleuthkit.autopsy.events.AutopsyEvent;
+import org.sleuthkit.datamodel.OsAccountInstance;
 
 /**
  * Listen for case events and update entries in the Central Repository database
@@ -75,7 +77,8 @@ public final class CaseEventListener implements PropertyChangeListener {
             Case.Events.DATA_SOURCE_ADDED,
             Case.Events.TAG_DEFINITION_CHANGED,
             Case.Events.CURRENT_CASE,
-            Case.Events.DATA_SOURCE_NAME_CHANGED);
+            Case.Events.DATA_SOURCE_NAME_CHANGED,
+            Case.Events.OS_ACCT_INSTANCES_ADDED);
 
     public CaseEventListener() {
         jobProcessingExecutor = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setNameFormat(CASE_EVENT_THREAD_NAME).build());
@@ -130,6 +133,14 @@ public final class CaseEventListener implements PropertyChangeListener {
                 jobProcessingExecutor.submit(new DataSourceNameChangedTask(dbManager, evt));
             }
             break;
+            case OS_ACCT_INSTANCES_ADDED: {
+                // STUB, TO BE REPLACED
+                List<OsAccountInstance> osAcctInstances = ((OsAcctInstancesAddedEvent) evt).getOsAccountInstances();
+                for (OsAccountInstance instance : osAcctInstances) {
+                    LOGGER.log(Level.INFO, String.format("Received OS account instance added message (instance ID = %d)", instance.getInstanceId()));
+                }
+            }
+            break;
         }
     }
 

Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java

@@ -78,6 +78,7 @@ import org.sleuthkit.datamodel.DerivedFile;
 import org.sleuthkit.datamodel.EncodedFileOutputStream;
 import org.sleuthkit.datamodel.ReadContentInputStream;
 import org.sleuthkit.datamodel.Score;
+import org.sleuthkit.datamodel.SleuthkitCase.CaseDbTransaction;
 import org.sleuthkit.datamodel.TskCoreException;
 import org.sleuthkit.datamodel.TskData;
 
@@ -800,18 +801,19 @@ class SevenZipExtractor {
             // intermediate nodes since the order is not guaranteed
             try {
                 unpackedTree.updateOrAddFileToCaseRec(statusMap, archiveFilePath, parentAr, archiveFile, depthMap);
-                if (checkForIngestCancellation(archiveFile)) {
+                unpackedTree.commitCurrentTransaction();
-                    return false;
+            } catch (TskCoreException | NoCurrentCaseException ex) {
-                }
+                logger.log(Level.SEVERE, "Error populating complete derived file hierarchy from the unpacked dir structure", ex); //NON-NLS
-
+                //TODO decide if anything to cleanup, for now bailing               
-            } catch (TskCoreException | NoCurrentCaseException e) {
+                unpackedTree.rollbackCurrentTransaction();
-                logger.log(Level.SEVERE, "Error populating complete derived file hierarchy from the unpacked dir structure", e); //NON-NLS
+            }
-                //TODO decide if anything to cleanup, for now bailing
+                
+            if (checkForIngestCancellation(archiveFile)) {
+                return false;
             }
             
             // Get the new files to be added to the case.
             unpackedFiles = unpackedTree.getAllFileObjects();
-
         } catch (SevenZipException | IllegalArgumentException ex) {
             logger.log(Level.WARNING, "Error unpacking file: " + archiveFile, ex); //NON-NLS
             //inbox message
@@ -1269,6 +1271,15 @@ class SevenZipExtractor {
 
         final UnpackedNode rootNode;
         private int nodesProcessed = 0;
+            
+        // It is significantly faster to add the DerivedFiles to the case on a transaction,
+        // but we don't want to hold the transaction (and case write lock) for the entire 
+        // stage. Instead, we use the same transaction for MAX_TRANSACTION_SIZE database operations
+        // and then commit that transaction and start a new one, giving at least a short window
+        // for other processes.
+        private CaseDbTransaction currentTransaction = null;
+        private long transactionCounter = 0;
+        private final static long MAX_TRANSACTION_SIZE = 1000;
 
         /**
          *
@@ -1449,10 +1460,10 @@ class SevenZipExtractor {
                 String nameInDatabase = getKeyFromUnpackedNode(node, archiveFilePath);
                 ZipFileStatusWrapper existingFile = nameInDatabase == null ? null : statusMap.get(nameInDatabase);
                 if (existingFile == null) {
-                    df = fileManager.addDerivedFile(node.getFileName(), node.getLocalRelPath(), node.getSize(),
+                    df = Case.getCurrentCaseThrows().getSleuthkitCase().addDerivedFile(node.getFileName(), node.getLocalRelPath(), node.getSize(),
                             node.getCtime(), node.getCrtime(), node.getAtime(), node.getMtime(),
                             node.isIsFile(), node.getParent().getFile(), "", MODULE_NAME,
-                            "", "", TskData.EncodingType.XOR1);
+                            "", "", TskData.EncodingType.XOR1, getCurrentTransaction());
                     statusMap.put(getKeyAbstractFile(df), new ZipFileStatusWrapper(df, ZipFileStatus.EXISTS));
                 } else {
                     String key = getKeyAbstractFile(existingFile.getFile());
@@ -1463,10 +1474,10 @@ class SevenZipExtractor {
                     if (existingFile.getStatus() == ZipFileStatus.UPDATE) {
                         //if the we are updating a file and its mime type was octet-stream we want to re-type it
                         String mimeType = existingFile.getFile().getMIMEType().equalsIgnoreCase("application/octet-stream") ? null : existingFile.getFile().getMIMEType();
-                        df = fileManager.updateDerivedFile((DerivedFile) existingFile.getFile(), node.getLocalRelPath(), node.getSize(),
+                        df = Case.getCurrentCaseThrows().getSleuthkitCase().updateDerivedFile((DerivedFile) existingFile.getFile(), node.getLocalRelPath(), node.getSize(),
                                 node.getCtime(), node.getCrtime(), node.getAtime(), node.getMtime(),
                                 node.isIsFile(), mimeType, "", MODULE_NAME,
-                                "", "", TskData.EncodingType.XOR1);
+                                "", "", TskData.EncodingType.XOR1, existingFile.getFile().getParent(), getCurrentTransaction());
                     } else {
                         //ALREADY CURRENT - SKIP
                         statusMap.put(key, new ZipFileStatusWrapper(existingFile.getFile(), ZipFileStatus.SKIP));
@@ -1474,7 +1485,7 @@ class SevenZipExtractor {
                     }
                 }
                 node.setFile(df);
-            } catch (TskCoreException ex) {
+            } catch (TskCoreException | NoCurrentCaseException ex) {
                 logger.log(Level.SEVERE, "Error adding a derived file to db:" + node.getFileName(), ex); //NON-NLS
                 throw new TskCoreException(
                         NbBundle.getMessage(SevenZipExtractor.class, "EmbeddedFileExtractorIngestModule.ArchiveExtractor.UnpackedTree.exception.msg",
@@ -1518,6 +1529,71 @@ class SevenZipExtractor {
                 updateOrAddFileToCaseRec(child, fileManager, statusMap, getKeyFromUnpackedNode(node, archiveFilePath), parentAr, archiveFile, depthMap);
             }
         }
+        
+        /**
+         * Get the current transaction being used in updateOrAddFileToCaseRec().
+         * If there is no transaction, one will be started. After the
+         * transaction has been used MAX_TRANSACTION_SIZE, it will be committed and a
+         * new transaction will be opened.
+         * 
+         * @return The open transaction.
+         * 
+         * @throws TskCoreException 
+         */
+        private CaseDbTransaction getCurrentTransaction() throws TskCoreException {
+        
+            if (currentTransaction == null) {
+                startTransaction();
+            } 
+
+            if (transactionCounter > MAX_TRANSACTION_SIZE) {
+                commitCurrentTransaction();
+                startTransaction();
+            }
+
+            transactionCounter++;
+            return currentTransaction;
+        }
+    
+        /**
+         * Open a transaction.
+         * 
+         * @throws TskCoreException 
+         */
+        private void startTransaction() throws TskCoreException {
+            try {
+                currentTransaction = Case.getCurrentCaseThrows().getSleuthkitCase().beginTransaction();
+                transactionCounter = 0;
+            } catch (NoCurrentCaseException ex) {
+                throw new TskCoreException("Case is closed");
+            }
+        }
+    
+        /**
+         * Commit the current transaction.
+         * 
+         * @throws TskCoreException 
+         */
+        private void commitCurrentTransaction() throws TskCoreException {
+            if (currentTransaction != null) {
+                currentTransaction.commit();
+                currentTransaction = null;
+            }
+        }
+    
+        /**
+         * Rollback the current transaction.
+         */
+        private void rollbackCurrentTransaction() {
+            if (currentTransaction != null) {
+                try {
+                    currentTransaction.rollback();
+                    currentTransaction = null;
+                } catch (TskCoreException ex) {
+                    // Ignored
+                }
+            }
+        }
 
         /**
          * A node in the unpacked tree that represents a file or folder.

Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileType.java

@@ -133,9 +133,9 @@ class FileType implements Serializable {
      *
      * @return True or false.
      */
-    boolean matches(final AbstractFile file) {
+    boolean matches(final AbstractFile file, byte[] startOfFileBuffer, int bufLen) {
         for (Signature sig : this.signatures) {
-            if (!sig.containedIn(file)) {
+            if (!sig.containedIn(file, startOfFileBuffer, bufLen)) {
                 return false;
             }
         }
@@ -327,7 +327,7 @@ class FileType implements Serializable {
          *
          * @return True or false.
          */
-        boolean containedIn(final AbstractFile file) {
+        boolean containedIn(final AbstractFile file, byte[] startOfFileBuffer, int bufLen) {
             if (offset >= file.getSize()) {
                 return false; // File is too small, offset lies outside file.
             }
@@ -340,7 +340,17 @@ class FileType implements Serializable {
             }
             try {
                 byte[] buffer = new byte[signatureBytes.length];
-                int bytesRead = file.read(buffer, actualOffset, signatureBytes.length);
+                int bytesRead;
+                if (actualOffset + signatureBytes.length < bufLen) {
+                    // The signature is contained in the buffer we've already read, so
+                    // just copy the appropriate section.
+                    for (int i = 0; i < signatureBytes.length;i++) {
+                        buffer[i] = startOfFileBuffer[(int)actualOffset + i];
+                    }
+                    bytesRead = signatureBytes.length;
+                } else {
+                    bytesRead = file.read(buffer, actualOffset, signatureBytes.length);
+                }
                 return ((bytesRead == signatureBytes.length) && (Arrays.equals(buffer, signatureBytes)));
             } catch (TskCoreException ex) {
                 /**

Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeDetector.java

@@ -51,7 +51,8 @@ public class FileTypeDetector {
     private final List<FileType> userDefinedFileTypes;
     private final List<FileType> autopsyDefinedFileTypes;
     private static SortedSet<String> tikaDetectedTypes;
-
+    private final int defaultBufferSize = 600; // Number of bytes to initially read from the file. Should cover most signatures.
+    
     /**
      * Gets a sorted set of the file types that can be detected: the MIME types
      * detected by Tika (without optional parameters), the custom MIME types
@@ -189,7 +190,6 @@ public class FileTypeDetector {
             // optional parameter attached.
             return removeOptionalParameter(mimeType);
         }
-
         /*
          * Mark non-regular files (refer to TskData.TSK_FS_META_TYPE_ENUM),
          * zero-sized files, unallocated space, and unused blocks (refer to
@@ -203,12 +203,24 @@ public class FileTypeDetector {
             mimeType = MimeTypes.OCTET_STREAM;
         }
 
+        /*
+         * Read in the beginning of the file and store it.
+         */
+        byte[] buf = new byte[defaultBufferSize];
+        int bufLen;
+        try {
+            bufLen = file.read(buf, 0, defaultBufferSize);
+        } catch (TskCoreException ex) {
+            // Proceed for now - the error will likely get logged next time the file is read.
+            bufLen = 0; 
+        }
+       
         /*
          * If the file is a regular file, give precedence to user-defined custom
          * file types.
          */
         if (null == mimeType) {
-            mimeType = detectUserDefinedType(file);
+            mimeType = detectUserDefinedType(file, buf, bufLen);
         }
 
         /*
@@ -216,7 +228,7 @@ public class FileTypeDetector {
          * custom file types defined by Autopsy.
          */
         if (null == mimeType) {
-            mimeType = detectAutopsyDefinedType(file);
+            mimeType = detectAutopsyDefinedType(file, buf, bufLen);
         }
 
         /*
@@ -296,7 +308,7 @@ public class FileTypeDetector {
          * Documented side effect: write the result to the AbstractFile object.
          */
         file.setMIMEType(mimeType);
-
+        
         return mimeType;
     }
 
@@ -352,11 +364,11 @@ public class FileTypeDetector {
      *
      * @return The MIME type as a string if a match is found; otherwise null.
      */
-    private String detectUserDefinedType(AbstractFile file) {
+    private String detectUserDefinedType(AbstractFile file, byte[] startOfFileBuffer, int bufLen) {
         String retValue = null;
 
         for (FileType fileType : userDefinedFileTypes) {
-            if (fileType.matches(file)) {
+            if (fileType.matches(file, startOfFileBuffer, bufLen)) {
                 retValue = fileType.getMimeType();
                 break;
             }
@@ -372,9 +384,9 @@ public class FileTypeDetector {
      *
      * @return The MIME type as a string if a match is found; otherwise null.
      */
-    private String detectAutopsyDefinedType(AbstractFile file) {
+    private String detectAutopsyDefinedType(AbstractFile file, byte[] startOfFileBuffer, int bufLen) {
         for (FileType fileType : autopsyDefinedFileTypes) {
-            if (fileType.matches(file)) {
+            if (fileType.matches(file, startOfFileBuffer, bufLen)) {
                 return fileType.getMimeType();
             }
         }

Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java

@@ -128,12 +128,42 @@ public class FileTypeIdIngestModule implements FileIngestModule {
      *                                  of CustomFileTypesManager.
      */
     private FileType detectUserDefinedFileType(AbstractFile file) throws CustomFileTypesManager.CustomFileTypesException {
+        
+        if (CustomFileTypesManager.getInstance().getUserDefinedFileTypes().isEmpty()) {
+            return null;
+        }
+        
+        /*
+         * Read in the beginning of the file once.
+         */
+        byte[] buf = new byte[1024];
+        int bufLen;
+        try {
+            bufLen = file.read(buf, 0, 1024);
+        } catch (TskCoreException ex) {
+            // Proceed for now - the error will likely get logged next time the file is read.
+            bufLen = 0; 
+        }
+        return detectUserDefinedFileType(file, buf, bufLen);
+    }
+
+    /**
+     * Determines whether or not a file matches a user-defined custom file type.
+     *
+     * @param file The file to test.
+     *
+     * @return The file type if a match is found; otherwise null.
+     *
+     * @throws CustomFileTypesException If there is an issue getting an instance
+     *                                  of CustomFileTypesManager.
+     */
+    private FileType detectUserDefinedFileType(AbstractFile file, byte[] startOfFileBuffer, int bufLen) throws CustomFileTypesManager.CustomFileTypesException {    
         FileType retValue = null;
 
         CustomFileTypesManager customFileTypesManager = CustomFileTypesManager.getInstance();
         List<FileType> fileTypesList = customFileTypesManager.getUserDefinedFileTypes();
         for (FileType fileType : fileTypesList) {
-            if (fileType.matches(file)) {
+            if (fileType.matches(file, startOfFileBuffer, bufLen)) {
                 retValue = fileType;
                 break;
             }