mstoeck3/autopsy-flatpak
1
0
Fork 0
mirror of https://github.com/overcuriousity/autopsy-flatpak.git synced 2025-07-14 08:56:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #7069 from sleuthkit/release-4.19.0

Browse Source 
Merge release 4.19.0 into develop
This commit is contained in:
Richard Cordovano 2021-06-24 14:28:13 -04:00 committed by GitHub
commit 7243c8caca
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 175 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
Core/src/org/sleuthkit/autopsy/casemodule/events/OsAcctInstancesAddedEvent.java
View File

@ -18,7 +18,6 @@
*/
package org.sleuthkit.autopsy.casemodule.events;
import java.util.ArrayList;
import java.util.List;
import static org.sleuthkit.autopsy.casemodule.Case.Events.OS_ACCT_INSTANCES_ADDED;
import org.sleuthkit.datamodel.OsAccountInstance;
@ -26,16 +25,16 @@ import org.sleuthkit.datamodel.SleuthkitCase;
import org.sleuthkit.datamodel.TskCoreException;
/**
* An application event published when OS accounts are added to the Sleuth Kit
* data model for a case.
* An application event published when OS account instances are added to the
* Sleuth Kit data model for a case.
*/
public final class OsAcctInstancesAddedEvent extends TskDataModelChangedEvent<OsAccountInstance, OsAccountInstance> {
private static final long serialVersionUID = 1L;
/**
* Constructs an application event published when OS account instances are added to
* the Sleuth Kit data model for a case.
* Constructs an application event published when OS account instances are
* added to the Sleuth Kit data model for a case.
*
* @param osAcctInstances The OS account instances that were added.
*/
@ -44,9 +43,9 @@ public final class OsAcctInstancesAddedEvent extends TskDataModelChangedEvent<Os
}
/**
* Gets the OS accounts that have been added or updated.
* Gets the OS account instances that have been added.
*
* @return The OS accounts.
* @return The OS account instances.
*/
public List<OsAccountInstance> getOsAccountInstances() {
return getNewValue();
@ -54,11 +53,7 @@ public final class OsAcctInstancesAddedEvent extends TskDataModelChangedEvent<Os
@Override
protected List<OsAccountInstance> getNewValueObjects(SleuthkitCase caseDb, List<Long> ids) throws TskCoreException {
List<OsAccountInstance> osAccountInstances = new ArrayList<>();
for (Long id : ids) {
//RJCTODO
}
return osAccountInstances;
return caseDb.getOsAccountManager().getOsAccountInstances(ids);
}
}

13
Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/CaseEventListener.java
View File

@ -37,6 +37,7 @@ import org.sleuthkit.autopsy.casemodule.events.ContentTagAddedEvent;
import org.sleuthkit.autopsy.casemodule.events.ContentTagDeletedEvent;
import org.sleuthkit.autopsy.casemodule.events.DataSourceAddedEvent;
import org.sleuthkit.autopsy.casemodule.events.DataSourceNameChangedEvent;
import org.sleuthkit.autopsy.casemodule.events.OsAcctInstancesAddedEvent;
import org.sleuthkit.autopsy.casemodule.services.TagsManager;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeInstance;
@ -56,6 +57,7 @@ import org.sleuthkit.datamodel.TskData;
import org.sleuthkit.autopsy.centralrepository.datamodel.CentralRepository;
import org.sleuthkit.datamodel.Tag;
import org.sleuthkit.autopsy.events.AutopsyEvent;
import org.sleuthkit.datamodel.OsAccountInstance;
/**
* Listen for case events and update entries in the Central Repository database
@ -75,7 +77,8 @@ public final class CaseEventListener implements PropertyChangeListener {
Case.Events.DATA_SOURCE_ADDED,
Case.Events.TAG_DEFINITION_CHANGED,
Case.Events.CURRENT_CASE,
Case.Events.DATA_SOURCE_NAME_CHANGED);
Case.Events.DATA_SOURCE_NAME_CHANGED,
Case.Events.OS_ACCT_INSTANCES_ADDED);
public CaseEventListener() {
jobProcessingExecutor = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setNameFormat(CASE_EVENT_THREAD_NAME).build());
@ -130,6 +133,14 @@ public final class CaseEventListener implements PropertyChangeListener {
jobProcessingExecutor.submit(new DataSourceNameChangedTask(dbManager, evt));
}
break;
case OS_ACCT_INSTANCES_ADDED: {
// STUB, TO BE REPLACED
List<OsAccountInstance> osAcctInstances = ((OsAcctInstancesAddedEvent) evt).getOsAccountInstances();
for (OsAccountInstance instance : osAcctInstances) {
LOGGER.log(Level.INFO, String.format("Received OS account instance added message (instance ID = %d)", instance.getInstanceId()));
}
}
break;
}
}

100
Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java
View File

@ -78,6 +78,7 @@ import org.sleuthkit.datamodel.DerivedFile;
import org.sleuthkit.datamodel.EncodedFileOutputStream;
import org.sleuthkit.datamodel.ReadContentInputStream;
import org.sleuthkit.datamodel.Score;
import org.sleuthkit.datamodel.SleuthkitCase.CaseDbTransaction;
import org.sleuthkit.datamodel.TskCoreException;
import org.sleuthkit.datamodel.TskData;
@ -800,18 +801,19 @@ class SevenZipExtractor {
// intermediate nodes since the order is not guaranteed
try {
unpackedTree.updateOrAddFileToCaseRec(statusMap, archiveFilePath, parentAr, archiveFile, depthMap);
if (checkForIngestCancellation(archiveFile)) {
return false;
}
} catch (TskCoreException | NoCurrentCaseException e) {
logger.log(Level.SEVERE, "Error populating complete derived file hierarchy from the unpacked dir structure", e); //NON-NLS
unpackedTree.commitCurrentTransaction();
} catch (TskCoreException | NoCurrentCaseException ex) {
logger.log(Level.SEVERE, "Error populating complete derived file hierarchy from the unpacked dir structure", ex); //NON-NLS
//TODO decide if anything to cleanup, for now bailing
unpackedTree.rollbackCurrentTransaction();
}
if (checkForIngestCancellation(archiveFile)) {
return false;
}
// Get the new files to be added to the case.
unpackedFiles = unpackedTree.getAllFileObjects();
} catch (SevenZipException | IllegalArgumentException ex) {
logger.log(Level.WARNING, "Error unpacking file: " + archiveFile, ex); //NON-NLS
//inbox message
@ -1270,6 +1272,15 @@ class SevenZipExtractor {
final UnpackedNode rootNode;
private int nodesProcessed = 0;
// It is significantly faster to add the DerivedFiles to the case on a transaction,
// but we don't want to hold the transaction (and case write lock) for the entire
// stage. Instead, we use the same transaction for MAX_TRANSACTION_SIZE database operations
// and then commit that transaction and start a new one, giving at least a short window
// for other processes.
private CaseDbTransaction currentTransaction = null;
private long transactionCounter = 0;
private final static long MAX_TRANSACTION_SIZE = 1000;
/**
*
* @param localPathRoot Path in module output folder that files will be
@ -1449,10 +1460,10 @@ class SevenZipExtractor {
String nameInDatabase = getKeyFromUnpackedNode(node, archiveFilePath);
ZipFileStatusWrapper existingFile = nameInDatabase == null ? null : statusMap.get(nameInDatabase);
if (existingFile == null) {
df = fileManager.addDerivedFile(node.getFileName(), node.getLocalRelPath(), node.getSize(),
df = Case.getCurrentCaseThrows().getSleuthkitCase().addDerivedFile(node.getFileName(), node.getLocalRelPath(), node.getSize(),
node.getCtime(), node.getCrtime(), node.getAtime(), node.getMtime(),
node.isIsFile(), node.getParent().getFile(), "", MODULE_NAME,
"", "", TskData.EncodingType.XOR1);
"", "", TskData.EncodingType.XOR1, getCurrentTransaction());
statusMap.put(getKeyAbstractFile(df), new ZipFileStatusWrapper(df, ZipFileStatus.EXISTS));
} else {
String key = getKeyAbstractFile(existingFile.getFile());
@ -1463,10 +1474,10 @@ class SevenZipExtractor {
if (existingFile.getStatus() == ZipFileStatus.UPDATE) {
//if the we are updating a file and its mime type was octet-stream we want to re-type it
String mimeType = existingFile.getFile().getMIMEType().equalsIgnoreCase("application/octet-stream") ? null : existingFile.getFile().getMIMEType();
df = fileManager.updateDerivedFile((DerivedFile) existingFile.getFile(), node.getLocalRelPath(), node.getSize(),
df = Case.getCurrentCaseThrows().getSleuthkitCase().updateDerivedFile((DerivedFile) existingFile.getFile(), node.getLocalRelPath(), node.getSize(),
node.getCtime(), node.getCrtime(), node.getAtime(), node.getMtime(),
node.isIsFile(), mimeType, "", MODULE_NAME,
"", "", TskData.EncodingType.XOR1);
"", "", TskData.EncodingType.XOR1, existingFile.getFile().getParent(), getCurrentTransaction());
} else {
//ALREADY CURRENT - SKIP
statusMap.put(key, new ZipFileStatusWrapper(existingFile.getFile(), ZipFileStatus.SKIP));
@ -1474,7 +1485,7 @@ class SevenZipExtractor {
}
}
node.setFile(df);
} catch (TskCoreException ex) {
} catch (TskCoreException | NoCurrentCaseException ex) {
logger.log(Level.SEVERE, "Error adding a derived file to db:" + node.getFileName(), ex); //NON-NLS
throw new TskCoreException(
NbBundle.getMessage(SevenZipExtractor.class, "EmbeddedFileExtractorIngestModule.ArchiveExtractor.UnpackedTree.exception.msg",
@ -1519,6 +1530,71 @@ class SevenZipExtractor {
}
}
/**
* Get the current transaction being used in updateOrAddFileToCaseRec().
* If there is no transaction, one will be started. After the
* transaction has been used MAX_TRANSACTION_SIZE, it will be committed and a
* new transaction will be opened.
*
* @return The open transaction.
*
* @throws TskCoreException
*/
private CaseDbTransaction getCurrentTransaction() throws TskCoreException {
if (currentTransaction == null) {
startTransaction();
}
if (transactionCounter > MAX_TRANSACTION_SIZE) {
commitCurrentTransaction();
startTransaction();
}
transactionCounter++;
return currentTransaction;
}
/**
* Open a transaction.
*
* @throws TskCoreException
*/
private void startTransaction() throws TskCoreException {
try {
currentTransaction = Case.getCurrentCaseThrows().getSleuthkitCase().beginTransaction();
transactionCounter = 0;
} catch (NoCurrentCaseException ex) {
throw new TskCoreException("Case is closed");
}
}
/**
* Commit the current transaction.
*
* @throws TskCoreException
*/
private void commitCurrentTransaction() throws TskCoreException {
if (currentTransaction != null) {
currentTransaction.commit();
currentTransaction = null;
}
}
/**
* Rollback the current transaction.
*/
private void rollbackCurrentTransaction() {
if (currentTransaction != null) {
try {
currentTransaction.rollback();
currentTransaction = null;
} catch (TskCoreException ex) {
// Ignored
}
}
}
/**
* A node in the unpacked tree that represents a file or folder.
*/

18
Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileType.java
View File

@ -133,9 +133,9 @@ class FileType implements Serializable {
*
* @return True or false.
*/
boolean matches(final AbstractFile file) {
boolean matches(final AbstractFile file, byte[] startOfFileBuffer, int bufLen) {
for (Signature sig : this.signatures) {
if (!sig.containedIn(file)) {
if (!sig.containedIn(file, startOfFileBuffer, bufLen)) {
return false;
}
}
@ -327,7 +327,7 @@ class FileType implements Serializable {
*
* @return True or false.
*/
boolean containedIn(final AbstractFile file) {
boolean containedIn(final AbstractFile file, byte[] startOfFileBuffer, int bufLen) {
if (offset >= file.getSize()) {
return false; // File is too small, offset lies outside file.
}
@ -340,7 +340,17 @@ class FileType implements Serializable {
}
try {
byte[] buffer = new byte[signatureBytes.length];
int bytesRead = file.read(buffer, actualOffset, signatureBytes.length);
int bytesRead;
if (actualOffset + signatureBytes.length < bufLen) {
// The signature is contained in the buffer we've already read, so
// just copy the appropriate section.
for (int i = 0; i < signatureBytes.length;i++) {
buffer[i] = startOfFileBuffer[(int)actualOffset + i];
}
bytesRead = signatureBytes.length;
} else {
bytesRead = file.read(buffer, actualOffset, signatureBytes.length);
}
return ((bytesRead == signatureBytes.length) && (Arrays.equals(buffer, signatureBytes)));
} catch (TskCoreException ex) {
/**

26
Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeDetector.java
View File

@ -51,6 +51,7 @@ public class FileTypeDetector {
private final List<FileType> userDefinedFileTypes;
private final List<FileType> autopsyDefinedFileTypes;
private static SortedSet<String> tikaDetectedTypes;
private final int defaultBufferSize = 600; // Number of bytes to initially read from the file. Should cover most signatures.
/**
* Gets a sorted set of the file types that can be detected: the MIME types
@ -189,7 +190,6 @@ public class FileTypeDetector {
// optional parameter attached.
return removeOptionalParameter(mimeType);
}
/*
* Mark non-regular files (refer to TskData.TSK_FS_META_TYPE_ENUM),
* zero-sized files, unallocated space, and unused blocks (refer to
@ -203,12 +203,24 @@ public class FileTypeDetector {
mimeType = MimeTypes.OCTET_STREAM;
}
/*
* Read in the beginning of the file and store it.
*/
byte[] buf = new byte[defaultBufferSize];
int bufLen;
try {
bufLen = file.read(buf, 0, defaultBufferSize);
} catch (TskCoreException ex) {
// Proceed for now - the error will likely get logged next time the file is read.
bufLen = 0;
}
/*
* If the file is a regular file, give precedence to user-defined custom
* file types.
*/
if (null == mimeType) {
mimeType = detectUserDefinedType(file);
mimeType = detectUserDefinedType(file, buf, bufLen);
}
/*
@ -216,7 +228,7 @@ public class FileTypeDetector {
* custom file types defined by Autopsy.
*/
if (null == mimeType) {
mimeType = detectAutopsyDefinedType(file);
mimeType = detectAutopsyDefinedType(file, buf, bufLen);
}
/*
@ -352,11 +364,11 @@ public class FileTypeDetector {
*
* @return The MIME type as a string if a match is found; otherwise null.
*/
private String detectUserDefinedType(AbstractFile file) {
private String detectUserDefinedType(AbstractFile file, byte[] startOfFileBuffer, int bufLen) {
String retValue = null;
for (FileType fileType : userDefinedFileTypes) {
if (fileType.matches(file)) {
if (fileType.matches(file, startOfFileBuffer, bufLen)) {
retValue = fileType.getMimeType();
break;
}
@ -372,9 +384,9 @@ public class FileTypeDetector {
*
* @return The MIME type as a string if a match is found; otherwise null.
*/
private String detectAutopsyDefinedType(AbstractFile file) {
private String detectAutopsyDefinedType(AbstractFile file, byte[] startOfFileBuffer, int bufLen) {
for (FileType fileType : autopsyDefinedFileTypes) {
if (fileType.matches(file)) {
if (fileType.matches(file, startOfFileBuffer, bufLen)) {
return fileType.getMimeType();
}
}

32
Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java
View File

@ -128,12 +128,42 @@ public class FileTypeIdIngestModule implements FileIngestModule {
* of CustomFileTypesManager.
*/
private FileType detectUserDefinedFileType(AbstractFile file) throws CustomFileTypesManager.CustomFileTypesException {
if (CustomFileTypesManager.getInstance().getUserDefinedFileTypes().isEmpty()) {
return null;
}
/*
* Read in the beginning of the file once.
*/
byte[] buf = new byte[1024];
int bufLen;
try {
bufLen = file.read(buf, 0, 1024);
} catch (TskCoreException ex) {
// Proceed for now - the error will likely get logged next time the file is read.
bufLen = 0;
}
return detectUserDefinedFileType(file, buf, bufLen);
}
/**
* Determines whether or not a file matches a user-defined custom file type.
*
* @param file The file to test.
*
* @return The file type if a match is found; otherwise null.
*
* @throws CustomFileTypesException If there is an issue getting an instance
* of CustomFileTypesManager.
*/
private FileType detectUserDefinedFileType(AbstractFile file, byte[] startOfFileBuffer, int bufLen) throws CustomFileTypesManager.CustomFileTypesException {
FileType retValue = null;
CustomFileTypesManager customFileTypesManager = CustomFileTypesManager.getInstance();
List<FileType> fileTypesList = customFileTypesManager.getUserDefinedFileTypes();
for (FileType fileType : fileTypesList) {
if (fileType.matches(file)) {
if (fileType.matches(file, startOfFileBuffer, bufLen)) {
retValue = fileType;
break;
}