Merge pull request #7056 from APriestman/7579_testTransaction

7579 Use transaction for adding derived files

Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java

@@ -78,6 +78,7 @@ import org.sleuthkit.datamodel.DerivedFile;
 import org.sleuthkit.datamodel.EncodedFileOutputStream;
 import org.sleuthkit.datamodel.ReadContentInputStream;
 import org.sleuthkit.datamodel.Score;
+import org.sleuthkit.datamodel.SleuthkitCase.CaseDbTransaction;
 import org.sleuthkit.datamodel.TskCoreException;
 import org.sleuthkit.datamodel.TskData;
 
@@ -800,18 +801,19 @@ class SevenZipExtractor {
             // intermediate nodes since the order is not guaranteed
             try {
                 unpackedTree.updateOrAddFileToCaseRec(statusMap, archiveFilePath, parentAr, archiveFile, depthMap);
+                unpackedTree.commitCurrentTransaction();
+            } catch (TskCoreException | NoCurrentCaseException ex) {
+                logger.log(Level.SEVERE, "Error populating complete derived file hierarchy from the unpacked dir structure", ex); //NON-NLS
+                //TODO decide if anything to cleanup, for now bailing               
+                unpackedTree.rollbackCurrentTransaction();
+            }
+                
             if (checkForIngestCancellation(archiveFile)) {
                 return false;
             }
             
-            } catch (TskCoreException | NoCurrentCaseException e) {
-                logger.log(Level.SEVERE, "Error populating complete derived file hierarchy from the unpacked dir structure", e); //NON-NLS
-                //TODO decide if anything to cleanup, for now bailing
-            }
-            
             // Get the new files to be added to the case.
             unpackedFiles = unpackedTree.getAllFileObjects();
-
         } catch (SevenZipException | IllegalArgumentException ex) {
             logger.log(Level.WARNING, "Error unpacking file: " + archiveFile, ex); //NON-NLS
             //inbox message
@@ -1270,6 +1272,15 @@ class SevenZipExtractor {
         final UnpackedNode rootNode;
         private int nodesProcessed = 0;
             
+        // It is significantly faster to add the DerivedFiles to the case on a transaction,
+        // but we don't want to hold the transaction (and case write lock) for the entire 
+        // stage. Instead, we use the same transaction for MAX_TRANSACTION_SIZE database operations
+        // and then commit that transaction and start a new one, giving at least a short window
+        // for other processes.
+        private CaseDbTransaction currentTransaction = null;
+        private long transactionCounter = 0;
+        private final static long MAX_TRANSACTION_SIZE = 1000;
+
         /**
          *
          * @param localPathRoot Path in module output folder that files will be
@@ -1449,10 +1460,10 @@ class SevenZipExtractor {
                 String nameInDatabase = getKeyFromUnpackedNode(node, archiveFilePath);
                 ZipFileStatusWrapper existingFile = nameInDatabase == null ? null : statusMap.get(nameInDatabase);
                 if (existingFile == null) {
-                    df = fileManager.addDerivedFile(node.getFileName(), node.getLocalRelPath(), node.getSize(),
+                    df = Case.getCurrentCaseThrows().getSleuthkitCase().addDerivedFile(node.getFileName(), node.getLocalRelPath(), node.getSize(),
                             node.getCtime(), node.getCrtime(), node.getAtime(), node.getMtime(),
                             node.isIsFile(), node.getParent().getFile(), "", MODULE_NAME,
-                            "", "", TskData.EncodingType.XOR1);
+                            "", "", TskData.EncodingType.XOR1, getCurrentTransaction());
                     statusMap.put(getKeyAbstractFile(df), new ZipFileStatusWrapper(df, ZipFileStatus.EXISTS));
                 } else {
                     String key = getKeyAbstractFile(existingFile.getFile());
@@ -1463,10 +1474,10 @@ class SevenZipExtractor {
                     if (existingFile.getStatus() == ZipFileStatus.UPDATE) {
                         //if the we are updating a file and its mime type was octet-stream we want to re-type it
                         String mimeType = existingFile.getFile().getMIMEType().equalsIgnoreCase("application/octet-stream") ? null : existingFile.getFile().getMIMEType();
-                        df = fileManager.updateDerivedFile((DerivedFile) existingFile.getFile(), node.getLocalRelPath(), node.getSize(),
+                        df = Case.getCurrentCaseThrows().getSleuthkitCase().updateDerivedFile((DerivedFile) existingFile.getFile(), node.getLocalRelPath(), node.getSize(),
                                 node.getCtime(), node.getCrtime(), node.getAtime(), node.getMtime(),
                                 node.isIsFile(), mimeType, "", MODULE_NAME,
-                                "", "", TskData.EncodingType.XOR1);
+                                "", "", TskData.EncodingType.XOR1, existingFile.getFile().getParent(), getCurrentTransaction());
                     } else {
                         //ALREADY CURRENT - SKIP
                         statusMap.put(key, new ZipFileStatusWrapper(existingFile.getFile(), ZipFileStatus.SKIP));
@@ -1474,7 +1485,7 @@ class SevenZipExtractor {
                     }
                 }
                 node.setFile(df);
-            } catch (TskCoreException ex) {
+            } catch (TskCoreException | NoCurrentCaseException ex) {
                 logger.log(Level.SEVERE, "Error adding a derived file to db:" + node.getFileName(), ex); //NON-NLS
                 throw new TskCoreException(
                         NbBundle.getMessage(SevenZipExtractor.class, "EmbeddedFileExtractorIngestModule.ArchiveExtractor.UnpackedTree.exception.msg",
@@ -1519,6 +1530,71 @@ class SevenZipExtractor {
             }
         }
         
+        /**
+         * Get the current transaction being used in updateOrAddFileToCaseRec().
+         * If there is no transaction, one will be started. After the
+         * transaction has been used MAX_TRANSACTION_SIZE, it will be committed and a
+         * new transaction will be opened.
+         * 
+         * @return The open transaction.
+         * 
+         * @throws TskCoreException 
+         */
+        private CaseDbTransaction getCurrentTransaction() throws TskCoreException {
+        
+            if (currentTransaction == null) {
+                startTransaction();
+            } 
+
+            if (transactionCounter > MAX_TRANSACTION_SIZE) {
+                commitCurrentTransaction();
+                startTransaction();
+            }
+
+            transactionCounter++;
+            return currentTransaction;
+        }
+    
+        /**
+         * Open a transaction.
+         * 
+         * @throws TskCoreException 
+         */
+        private void startTransaction() throws TskCoreException {
+            try {
+                currentTransaction = Case.getCurrentCaseThrows().getSleuthkitCase().beginTransaction();
+                transactionCounter = 0;
+            } catch (NoCurrentCaseException ex) {
+                throw new TskCoreException("Case is closed");
+            }
+        }
+    
+        /**
+         * Commit the current transaction.
+         * 
+         * @throws TskCoreException 
+         */
+        private void commitCurrentTransaction() throws TskCoreException {
+            if (currentTransaction != null) {
+                currentTransaction.commit();
+                currentTransaction = null;
+            }
+        }
+    
+        /**
+         * Rollback the current transaction.
+         */
+        private void rollbackCurrentTransaction() {
+            if (currentTransaction != null) {
+                try {
+                    currentTransaction.rollback();
+                    currentTransaction = null;
+                } catch (TskCoreException ex) {
+                    // Ignored
+                }
+            }
+        }
+
         /**
          * A node in the unpacked tree that represents a file or folder.
          */