mstoeck3/autopsy-flatpak
1
0
Fork 0
mirror of https://github.com/overcuriousity/autopsy-flatpak.git synced 2025-07-17 18:17:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #7056 from APriestman/7579_testTransaction

Browse Source 
7579 Use transaction for adding derived files
This commit is contained in:
Richard Cordovano 2021-06-23 15:35:41 -04:00 committed by GitHub
commit badb36a43e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 89 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

102
Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java
View File

@ -78,6 +78,7 @@ import org.sleuthkit.datamodel.DerivedFile;
import org.sleuthkit.datamodel.EncodedFileOutputStream;
import org.sleuthkit.datamodel.ReadContentInputStream;
import org.sleuthkit.datamodel.Score;
import org.sleuthkit.datamodel.SleuthkitCase.CaseDbTransaction;
import org.sleuthkit.datamodel.TskCoreException;
import org.sleuthkit.datamodel.TskData;
@ -800,18 +801,19 @@ class SevenZipExtractor {
// intermediate nodes since the order is not guaranteed
try {
unpackedTree.updateOrAddFileToCaseRec(statusMap, archiveFilePath, parentAr, archiveFile, depthMap);
if (checkForIngestCancellation(archiveFile)) {
return false;
}
} catch (TskCoreException | NoCurrentCaseException e) {
logger.log(Level.SEVERE, "Error populating complete derived file hierarchy from the unpacked dir structure", e); //NON-NLS
//TODO decide if anything to cleanup, for now bailing
unpackedTree.commitCurrentTransaction();
} catch (TskCoreException | NoCurrentCaseException ex) {
logger.log(Level.SEVERE, "Error populating complete derived file hierarchy from the unpacked dir structure", ex); //NON-NLS
//TODO decide if anything to cleanup, for now bailing
unpackedTree.rollbackCurrentTransaction();
}
if (checkForIngestCancellation(archiveFile)) {
return false;
}
// Get the new files to be added to the case.
unpackedFiles = unpackedTree.getAllFileObjects();
} catch (SevenZipException | IllegalArgumentException ex) {
logger.log(Level.WARNING, "Error unpacking file: " + archiveFile, ex); //NON-NLS
//inbox message
@ -1269,6 +1271,15 @@ class SevenZipExtractor {
final UnpackedNode rootNode;
private int nodesProcessed = 0;
// It is significantly faster to add the DerivedFiles to the case on a transaction,
// but we don't want to hold the transaction (and case write lock) for the entire
// stage. Instead, we use the same transaction for MAX_TRANSACTION_SIZE database operations
// and then commit that transaction and start a new one, giving at least a short window
// for other processes.
private CaseDbTransaction currentTransaction = null;
private long transactionCounter = 0;
private final static long MAX_TRANSACTION_SIZE = 1000;
/**
*
@ -1449,10 +1460,10 @@ class SevenZipExtractor {
String nameInDatabase = getKeyFromUnpackedNode(node, archiveFilePath);
ZipFileStatusWrapper existingFile = nameInDatabase == null ? null : statusMap.get(nameInDatabase);
if (existingFile == null) {
df = fileManager.addDerivedFile(node.getFileName(), node.getLocalRelPath(), node.getSize(),
df = Case.getCurrentCaseThrows().getSleuthkitCase().addDerivedFile(node.getFileName(), node.getLocalRelPath(), node.getSize(),
node.getCtime(), node.getCrtime(), node.getAtime(), node.getMtime(),
node.isIsFile(), node.getParent().getFile(), "", MODULE_NAME,
"", "", TskData.EncodingType.XOR1);
"", "", TskData.EncodingType.XOR1, getCurrentTransaction());
statusMap.put(getKeyAbstractFile(df), new ZipFileStatusWrapper(df, ZipFileStatus.EXISTS));
} else {
String key = getKeyAbstractFile(existingFile.getFile());
@ -1463,10 +1474,10 @@ class SevenZipExtractor {
if (existingFile.getStatus() == ZipFileStatus.UPDATE) {
//if the we are updating a file and its mime type was octet-stream we want to re-type it
String mimeType = existingFile.getFile().getMIMEType().equalsIgnoreCase("application/octet-stream") ? null : existingFile.getFile().getMIMEType();
df = fileManager.updateDerivedFile((DerivedFile) existingFile.getFile(), node.getLocalRelPath(), node.getSize(),
df = Case.getCurrentCaseThrows().getSleuthkitCase().updateDerivedFile((DerivedFile) existingFile.getFile(), node.getLocalRelPath(), node.getSize(),
node.getCtime(), node.getCrtime(), node.getAtime(), node.getMtime(),
node.isIsFile(), mimeType, "", MODULE_NAME,
"", "", TskData.EncodingType.XOR1);
"", "", TskData.EncodingType.XOR1, existingFile.getFile().getParent(), getCurrentTransaction());
} else {
//ALREADY CURRENT - SKIP
statusMap.put(key, new ZipFileStatusWrapper(existingFile.getFile(), ZipFileStatus.SKIP));
@ -1474,7 +1485,7 @@ class SevenZipExtractor {
}
}
node.setFile(df);
} catch (TskCoreException ex) {
} catch (TskCoreException | NoCurrentCaseException ex) {
logger.log(Level.SEVERE, "Error adding a derived file to db:" + node.getFileName(), ex); //NON-NLS
throw new TskCoreException(
NbBundle.getMessage(SevenZipExtractor.class, "EmbeddedFileExtractorIngestModule.ArchiveExtractor.UnpackedTree.exception.msg",
@ -1518,6 +1529,71 @@ class SevenZipExtractor {
updateOrAddFileToCaseRec(child, fileManager, statusMap, getKeyFromUnpackedNode(node, archiveFilePath), parentAr, archiveFile, depthMap);
}
}
/**
* Get the current transaction being used in updateOrAddFileToCaseRec().
* If there is no transaction, one will be started. After the
* transaction has been used MAX_TRANSACTION_SIZE, it will be committed and a
* new transaction will be opened.
*
* @return The open transaction.
*
* @throws TskCoreException
*/
private CaseDbTransaction getCurrentTransaction() throws TskCoreException {
if (currentTransaction == null) {
startTransaction();
}
if (transactionCounter > MAX_TRANSACTION_SIZE) {
commitCurrentTransaction();
startTransaction();
}
transactionCounter++;
return currentTransaction;
}
/**
* Open a transaction.
*
* @throws TskCoreException
*/
private void startTransaction() throws TskCoreException {
try {
currentTransaction = Case.getCurrentCaseThrows().getSleuthkitCase().beginTransaction();
transactionCounter = 0;
} catch (NoCurrentCaseException ex) {
throw new TskCoreException("Case is closed");
}
}
/**
* Commit the current transaction.
*
* @throws TskCoreException
*/
private void commitCurrentTransaction() throws TskCoreException {
if (currentTransaction != null) {
currentTransaction.commit();
currentTransaction = null;
}
}
/**
* Rollback the current transaction.
*/
private void rollbackCurrentTransaction() {
if (currentTransaction != null) {
try {
currentTransaction.rollback();
currentTransaction = null;
} catch (TskCoreException ex) {
// Ignored
}
}
}
/**
* A node in the unpacked tree that represents a file or folder.