diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/events/OsAcctInstancesAddedEvent.java b/Core/src/org/sleuthkit/autopsy/casemodule/events/OsAcctInstancesAddedEvent.java index 17b8f31e9a..d752bf27fb 100755 --- a/Core/src/org/sleuthkit/autopsy/casemodule/events/OsAcctInstancesAddedEvent.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/events/OsAcctInstancesAddedEvent.java @@ -18,7 +18,6 @@ */ package org.sleuthkit.autopsy.casemodule.events; -import java.util.ArrayList; import java.util.List; import static org.sleuthkit.autopsy.casemodule.Case.Events.OS_ACCT_INSTANCES_ADDED; import org.sleuthkit.datamodel.OsAccountInstance; @@ -26,16 +25,16 @@ import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.TskCoreException; /** - * An application event published when OS accounts are added to the Sleuth Kit - * data model for a case. + * An application event published when OS account instances are added to the + * Sleuth Kit data model for a case. */ public final class OsAcctInstancesAddedEvent extends TskDataModelChangedEvent { private static final long serialVersionUID = 1L; /** - * Constructs an application event published when OS account instances are added to - * the Sleuth Kit data model for a case. + * Constructs an application event published when OS account instances are + * added to the Sleuth Kit data model for a case. * * @param osAcctInstances The OS account instances that were added. */ @@ -44,9 +43,9 @@ public final class OsAcctInstancesAddedEvent extends TskDataModelChangedEvent getOsAccountInstances() { return getNewValue(); @@ -54,11 +53,7 @@ public final class OsAcctInstancesAddedEvent extends TskDataModelChangedEvent getNewValueObjects(SleuthkitCase caseDb, List ids) throws TskCoreException { - List osAccountInstances = new ArrayList<>(); - for (Long id : ids) { - //RJCTODO - } - return osAccountInstances; + return caseDb.getOsAccountManager().getOsAccountInstances(ids); } -} \ No newline at end of file +} diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/CaseEventListener.java b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/CaseEventListener.java index 806d18c63f..218a0101e8 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/CaseEventListener.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/CaseEventListener.java @@ -37,6 +37,7 @@ import org.sleuthkit.autopsy.casemodule.events.ContentTagAddedEvent; import org.sleuthkit.autopsy.casemodule.events.ContentTagDeletedEvent; import org.sleuthkit.autopsy.casemodule.events.DataSourceAddedEvent; import org.sleuthkit.autopsy.casemodule.events.DataSourceNameChangedEvent; +import org.sleuthkit.autopsy.casemodule.events.OsAcctInstancesAddedEvent; import org.sleuthkit.autopsy.casemodule.services.TagsManager; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeInstance; @@ -56,6 +57,7 @@ import org.sleuthkit.datamodel.TskData; import org.sleuthkit.autopsy.centralrepository.datamodel.CentralRepository; import org.sleuthkit.datamodel.Tag; import org.sleuthkit.autopsy.events.AutopsyEvent; +import org.sleuthkit.datamodel.OsAccountInstance; /** * Listen for case events and update entries in the Central Repository database @@ -75,7 +77,8 @@ public final class CaseEventListener implements PropertyChangeListener { Case.Events.DATA_SOURCE_ADDED, Case.Events.TAG_DEFINITION_CHANGED, Case.Events.CURRENT_CASE, - Case.Events.DATA_SOURCE_NAME_CHANGED); + Case.Events.DATA_SOURCE_NAME_CHANGED, + Case.Events.OS_ACCT_INSTANCES_ADDED); public CaseEventListener() { jobProcessingExecutor = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setNameFormat(CASE_EVENT_THREAD_NAME).build()); @@ -130,6 +133,14 @@ public final class CaseEventListener implements PropertyChangeListener { jobProcessingExecutor.submit(new DataSourceNameChangedTask(dbManager, evt)); } break; + case OS_ACCT_INSTANCES_ADDED: { + // STUB, TO BE REPLACED + List osAcctInstances = ((OsAcctInstancesAddedEvent) evt).getOsAccountInstances(); + for (OsAccountInstance instance : osAcctInstances) { + LOGGER.log(Level.INFO, String.format("Received OS account instance added message (instance ID = %d)", instance.getInstanceId())); + } + } + break; } } diff --git a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java index 6fa1e93c5a..273737ff18 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java +++ b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java @@ -78,6 +78,7 @@ import org.sleuthkit.datamodel.DerivedFile; import org.sleuthkit.datamodel.EncodedFileOutputStream; import org.sleuthkit.datamodel.ReadContentInputStream; import org.sleuthkit.datamodel.Score; +import org.sleuthkit.datamodel.SleuthkitCase.CaseDbTransaction; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskData; @@ -800,18 +801,19 @@ class SevenZipExtractor { // intermediate nodes since the order is not guaranteed try { unpackedTree.updateOrAddFileToCaseRec(statusMap, archiveFilePath, parentAr, archiveFile, depthMap); - if (checkForIngestCancellation(archiveFile)) { - return false; - } - - } catch (TskCoreException | NoCurrentCaseException e) { - logger.log(Level.SEVERE, "Error populating complete derived file hierarchy from the unpacked dir structure", e); //NON-NLS - //TODO decide if anything to cleanup, for now bailing + unpackedTree.commitCurrentTransaction(); + } catch (TskCoreException | NoCurrentCaseException ex) { + logger.log(Level.SEVERE, "Error populating complete derived file hierarchy from the unpacked dir structure", ex); //NON-NLS + //TODO decide if anything to cleanup, for now bailing + unpackedTree.rollbackCurrentTransaction(); + } + + if (checkForIngestCancellation(archiveFile)) { + return false; } // Get the new files to be added to the case. unpackedFiles = unpackedTree.getAllFileObjects(); - } catch (SevenZipException | IllegalArgumentException ex) { logger.log(Level.WARNING, "Error unpacking file: " + archiveFile, ex); //NON-NLS //inbox message @@ -1269,6 +1271,15 @@ class SevenZipExtractor { final UnpackedNode rootNode; private int nodesProcessed = 0; + + // It is significantly faster to add the DerivedFiles to the case on a transaction, + // but we don't want to hold the transaction (and case write lock) for the entire + // stage. Instead, we use the same transaction for MAX_TRANSACTION_SIZE database operations + // and then commit that transaction and start a new one, giving at least a short window + // for other processes. + private CaseDbTransaction currentTransaction = null; + private long transactionCounter = 0; + private final static long MAX_TRANSACTION_SIZE = 1000; /** * @@ -1449,10 +1460,10 @@ class SevenZipExtractor { String nameInDatabase = getKeyFromUnpackedNode(node, archiveFilePath); ZipFileStatusWrapper existingFile = nameInDatabase == null ? null : statusMap.get(nameInDatabase); if (existingFile == null) { - df = fileManager.addDerivedFile(node.getFileName(), node.getLocalRelPath(), node.getSize(), + df = Case.getCurrentCaseThrows().getSleuthkitCase().addDerivedFile(node.getFileName(), node.getLocalRelPath(), node.getSize(), node.getCtime(), node.getCrtime(), node.getAtime(), node.getMtime(), node.isIsFile(), node.getParent().getFile(), "", MODULE_NAME, - "", "", TskData.EncodingType.XOR1); + "", "", TskData.EncodingType.XOR1, getCurrentTransaction()); statusMap.put(getKeyAbstractFile(df), new ZipFileStatusWrapper(df, ZipFileStatus.EXISTS)); } else { String key = getKeyAbstractFile(existingFile.getFile()); @@ -1463,10 +1474,10 @@ class SevenZipExtractor { if (existingFile.getStatus() == ZipFileStatus.UPDATE) { //if the we are updating a file and its mime type was octet-stream we want to re-type it String mimeType = existingFile.getFile().getMIMEType().equalsIgnoreCase("application/octet-stream") ? null : existingFile.getFile().getMIMEType(); - df = fileManager.updateDerivedFile((DerivedFile) existingFile.getFile(), node.getLocalRelPath(), node.getSize(), + df = Case.getCurrentCaseThrows().getSleuthkitCase().updateDerivedFile((DerivedFile) existingFile.getFile(), node.getLocalRelPath(), node.getSize(), node.getCtime(), node.getCrtime(), node.getAtime(), node.getMtime(), node.isIsFile(), mimeType, "", MODULE_NAME, - "", "", TskData.EncodingType.XOR1); + "", "", TskData.EncodingType.XOR1, existingFile.getFile().getParent(), getCurrentTransaction()); } else { //ALREADY CURRENT - SKIP statusMap.put(key, new ZipFileStatusWrapper(existingFile.getFile(), ZipFileStatus.SKIP)); @@ -1474,7 +1485,7 @@ class SevenZipExtractor { } } node.setFile(df); - } catch (TskCoreException ex) { + } catch (TskCoreException | NoCurrentCaseException ex) { logger.log(Level.SEVERE, "Error adding a derived file to db:" + node.getFileName(), ex); //NON-NLS throw new TskCoreException( NbBundle.getMessage(SevenZipExtractor.class, "EmbeddedFileExtractorIngestModule.ArchiveExtractor.UnpackedTree.exception.msg", @@ -1518,6 +1529,71 @@ class SevenZipExtractor { updateOrAddFileToCaseRec(child, fileManager, statusMap, getKeyFromUnpackedNode(node, archiveFilePath), parentAr, archiveFile, depthMap); } } + + /** + * Get the current transaction being used in updateOrAddFileToCaseRec(). + * If there is no transaction, one will be started. After the + * transaction has been used MAX_TRANSACTION_SIZE, it will be committed and a + * new transaction will be opened. + * + * @return The open transaction. + * + * @throws TskCoreException + */ + private CaseDbTransaction getCurrentTransaction() throws TskCoreException { + + if (currentTransaction == null) { + startTransaction(); + } + + if (transactionCounter > MAX_TRANSACTION_SIZE) { + commitCurrentTransaction(); + startTransaction(); + } + + transactionCounter++; + return currentTransaction; + } + + /** + * Open a transaction. + * + * @throws TskCoreException + */ + private void startTransaction() throws TskCoreException { + try { + currentTransaction = Case.getCurrentCaseThrows().getSleuthkitCase().beginTransaction(); + transactionCounter = 0; + } catch (NoCurrentCaseException ex) { + throw new TskCoreException("Case is closed"); + } + } + + /** + * Commit the current transaction. + * + * @throws TskCoreException + */ + private void commitCurrentTransaction() throws TskCoreException { + if (currentTransaction != null) { + currentTransaction.commit(); + currentTransaction = null; + } + } + + /** + * Rollback the current transaction. + */ + private void rollbackCurrentTransaction() { + if (currentTransaction != null) { + try { + currentTransaction.rollback(); + currentTransaction = null; + } catch (TskCoreException ex) { + // Ignored + } + } + } /** * A node in the unpacked tree that represents a file or folder. diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileType.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileType.java index acad818f09..2d8119b809 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileType.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileType.java @@ -133,9 +133,9 @@ class FileType implements Serializable { * * @return True or false. */ - boolean matches(final AbstractFile file) { + boolean matches(final AbstractFile file, byte[] startOfFileBuffer, int bufLen) { for (Signature sig : this.signatures) { - if (!sig.containedIn(file)) { + if (!sig.containedIn(file, startOfFileBuffer, bufLen)) { return false; } } @@ -327,7 +327,7 @@ class FileType implements Serializable { * * @return True or false. */ - boolean containedIn(final AbstractFile file) { + boolean containedIn(final AbstractFile file, byte[] startOfFileBuffer, int bufLen) { if (offset >= file.getSize()) { return false; // File is too small, offset lies outside file. } @@ -340,7 +340,17 @@ class FileType implements Serializable { } try { byte[] buffer = new byte[signatureBytes.length]; - int bytesRead = file.read(buffer, actualOffset, signatureBytes.length); + int bytesRead; + if (actualOffset + signatureBytes.length < bufLen) { + // The signature is contained in the buffer we've already read, so + // just copy the appropriate section. + for (int i = 0; i < signatureBytes.length;i++) { + buffer[i] = startOfFileBuffer[(int)actualOffset + i]; + } + bytesRead = signatureBytes.length; + } else { + bytesRead = file.read(buffer, actualOffset, signatureBytes.length); + } return ((bytesRead == signatureBytes.length) && (Arrays.equals(buffer, signatureBytes))); } catch (TskCoreException ex) { /** diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeDetector.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeDetector.java index edd61eb4d1..52b6d9e191 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeDetector.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeDetector.java @@ -51,7 +51,8 @@ public class FileTypeDetector { private final List userDefinedFileTypes; private final List autopsyDefinedFileTypes; private static SortedSet tikaDetectedTypes; - + private final int defaultBufferSize = 600; // Number of bytes to initially read from the file. Should cover most signatures. + /** * Gets a sorted set of the file types that can be detected: the MIME types * detected by Tika (without optional parameters), the custom MIME types @@ -189,7 +190,6 @@ public class FileTypeDetector { // optional parameter attached. return removeOptionalParameter(mimeType); } - /* * Mark non-regular files (refer to TskData.TSK_FS_META_TYPE_ENUM), * zero-sized files, unallocated space, and unused blocks (refer to @@ -203,12 +203,24 @@ public class FileTypeDetector { mimeType = MimeTypes.OCTET_STREAM; } + /* + * Read in the beginning of the file and store it. + */ + byte[] buf = new byte[defaultBufferSize]; + int bufLen; + try { + bufLen = file.read(buf, 0, defaultBufferSize); + } catch (TskCoreException ex) { + // Proceed for now - the error will likely get logged next time the file is read. + bufLen = 0; + } + /* * If the file is a regular file, give precedence to user-defined custom * file types. */ if (null == mimeType) { - mimeType = detectUserDefinedType(file); + mimeType = detectUserDefinedType(file, buf, bufLen); } /* @@ -216,7 +228,7 @@ public class FileTypeDetector { * custom file types defined by Autopsy. */ if (null == mimeType) { - mimeType = detectAutopsyDefinedType(file); + mimeType = detectAutopsyDefinedType(file, buf, bufLen); } /* @@ -296,7 +308,7 @@ public class FileTypeDetector { * Documented side effect: write the result to the AbstractFile object. */ file.setMIMEType(mimeType); - + return mimeType; } @@ -352,11 +364,11 @@ public class FileTypeDetector { * * @return The MIME type as a string if a match is found; otherwise null. */ - private String detectUserDefinedType(AbstractFile file) { + private String detectUserDefinedType(AbstractFile file, byte[] startOfFileBuffer, int bufLen) { String retValue = null; for (FileType fileType : userDefinedFileTypes) { - if (fileType.matches(file)) { + if (fileType.matches(file, startOfFileBuffer, bufLen)) { retValue = fileType.getMimeType(); break; } @@ -372,9 +384,9 @@ public class FileTypeDetector { * * @return The MIME type as a string if a match is found; otherwise null. */ - private String detectAutopsyDefinedType(AbstractFile file) { + private String detectAutopsyDefinedType(AbstractFile file, byte[] startOfFileBuffer, int bufLen) { for (FileType fileType : autopsyDefinedFileTypes) { - if (fileType.matches(file)) { + if (fileType.matches(file, startOfFileBuffer, bufLen)) { return fileType.getMimeType(); } } diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java index d5aa0d86fb..5dd6f49410 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java @@ -128,12 +128,42 @@ public class FileTypeIdIngestModule implements FileIngestModule { * of CustomFileTypesManager. */ private FileType detectUserDefinedFileType(AbstractFile file) throws CustomFileTypesManager.CustomFileTypesException { + + if (CustomFileTypesManager.getInstance().getUserDefinedFileTypes().isEmpty()) { + return null; + } + + /* + * Read in the beginning of the file once. + */ + byte[] buf = new byte[1024]; + int bufLen; + try { + bufLen = file.read(buf, 0, 1024); + } catch (TskCoreException ex) { + // Proceed for now - the error will likely get logged next time the file is read. + bufLen = 0; + } + return detectUserDefinedFileType(file, buf, bufLen); + } + + /** + * Determines whether or not a file matches a user-defined custom file type. + * + * @param file The file to test. + * + * @return The file type if a match is found; otherwise null. + * + * @throws CustomFileTypesException If there is an issue getting an instance + * of CustomFileTypesManager. + */ + private FileType detectUserDefinedFileType(AbstractFile file, byte[] startOfFileBuffer, int bufLen) throws CustomFileTypesManager.CustomFileTypesException { FileType retValue = null; CustomFileTypesManager customFileTypesManager = CustomFileTypesManager.getInstance(); List fileTypesList = customFileTypesManager.getUserDefinedFileTypes(); for (FileType fileType : fileTypesList) { - if (fileType.matches(file)) { + if (fileType.matches(file, startOfFileBuffer, bufLen)) { retValue = fileType; break; }