mstoeck3/autopsy-flatpak
1
0
Fork 0
mirror of https://github.com/overcuriousity/autopsy-flatpak.git synced 2025-07-09 14:49:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #4639 from sleuthkit/chrome_cache_refactor

Browse Source 
added comments, renamed. No logic changes except moving of an 'if'
This commit is contained in:
Richard Cordovano 2019-03-22 13:30:43 -04:00 committed by GitHub
commit 026fceaa62
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 165 additions and 141 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

166
RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ChromeCacheExtractor.java
View File

@ -75,7 +75,7 @@ import org.sleuthkit.datamodel.TskException;
*/ */
final class ChromeCacheExtractor { final class ChromeCacheExtractor {
private final static String DEFAULT_CACHE_STR = "default/cache"; //NON-NLS private final static String DEFAULT_CACHE_PATH_STR = "default/cache"; //NON-NLS
private final static String BROTLI_MIMETYPE ="application/x-brotli"; //NON-NLS private final static String BROTLI_MIMETYPE ="application/x-brotli"; //NON-NLS
private final static long UINT32_MASK = 0xFFFFFFFFl; private final static long UINT32_MASK = 0xFFFFFFFFl;
@ -99,7 +99,7 @@ final class ChromeCacheExtractor {
private FileManager fileManager; private FileManager fileManager;
// A file table to cache copies of index and data_n files. // A file table to cache copies of index and data_n files.
private final Map<String, CacheFileCopy> filesTable = new HashMap<>(); private final Map<String, CacheFileCopy> fileCopyCache = new HashMap<>();
// A file table to cache the f_* files. // A file table to cache the f_* files.
private final Map<String, AbstractFile> externalFilesTable = new HashMap<>(); private final Map<String, AbstractFile> externalFilesTable = new HashMap<>();
@ -152,7 +152,7 @@ final class ChromeCacheExtractor {
* *
* @throws IngestModuleException * @throws IngestModuleException
*/ */
void moduleInit() throws IngestModuleException { private void moduleInit() throws IngestModuleException {
try { try {
currentCase = Case.getCurrentCaseThrows(); currentCase = Case.getCurrentCaseThrows();
@ -179,9 +179,9 @@ final class ChromeCacheExtractor {
* *
* @throws org.sleuthkit.autopsy.ingest.IngestModule.IngestModuleException * @throws org.sleuthkit.autopsy.ingest.IngestModule.IngestModuleException
*/ */
void subInit(String cachePath) throws IngestModuleException { private void resetForNewFolder(String cachePath) throws IngestModuleException {
filesTable.clear(); fileCopyCache.clear();
externalFilesTable.clear(); externalFilesTable.clear();
String cacheAbsOutputFolderName = this.getAbsOutputFolderName() + cachePath; String cacheAbsOutputFolderName = this.getAbsOutputFolderName() + cachePath;
@ -203,9 +203,9 @@ final class ChromeCacheExtractor {
* Removes any temp copies of cache files created during extraction. * Removes any temp copies of cache files created during extraction.
* *
*/ */
void cleanup () { private void cleanup () {
for (Entry<String, CacheFileCopy> entry : this.filesTable.entrySet()) { for (Entry<String, CacheFileCopy> entry : this.fileCopyCache.entrySet()) {
Path tempFilePath = Paths.get(RAImageIngestModule.getRATempPath(currentCase, moduleName), entry.getKey() ); Path tempFilePath = Paths.get(RAImageIngestModule.getRATempPath(currentCase, moduleName), entry.getKey() );
try { try {
entry.getValue().getFileCopy().getChannel().close(); entry.getValue().getFileCopy().getChannel().close();
@ -240,7 +240,7 @@ final class ChromeCacheExtractor {
} }
/** /**
* Extracts the data from Chrome caches * Extracts the data from Chrome caches . Main entry point for the analysis
* *
* A data source may have multiple Chrome user profiles and caches. * A data source may have multiple Chrome user profiles and caches.
* *
@ -255,14 +255,14 @@ final class ChromeCacheExtractor {
return; return;
} }
// Find all possible caches // Find and process the cache folders. There could be one per user
List<AbstractFile> indexFiles; List<AbstractFile> indexFiles;
try { try {
indexFiles = findCacheFiles("index"); //NON-NLS indexFiles = findCacheIndexFiles();
// Get each of the caches // Process each of the caches
for (AbstractFile indexFile: indexFiles) { for (AbstractFile indexFile: indexFiles) {
getCache(indexFile); processCacheIndexFile(indexFile);
} }
} catch (TskCoreException ex) { } catch (TskCoreException ex) {
@ -272,24 +272,27 @@ final class ChromeCacheExtractor {
} }
/** /**
* Extracts the cache for the specified cache index file. * Processes a user's cache and creates corresponding artifacts and derived files.
* *
* @param cacheIndexFile * @param cacheIndexFile Cache index file for a given user
*/ */
void getCache(AbstractFile indexAbstractFile) { private void processCacheIndexFile(AbstractFile indexAbstractFile) {
String cachePath = indexAbstractFile.getParentPath(); String cachePath = indexAbstractFile.getParentPath();
Optional<CacheFileCopy> indexFile; Optional<CacheFileCopy> indexFileCopy;
try { try {
subInit(cachePath); resetForNewFolder(cachePath);
indexFile = this.getCacheFileCopy(indexAbstractFile.getName(), cachePath); // @@@ This is little ineffecient because we later in this call search for the AbstractFile that we currently have
if (!indexFile.isPresent()) { indexFileCopy = this.getCacheFileCopy(indexAbstractFile.getName(), cachePath);
if (!indexFileCopy.isPresent()) {
String msg = String.format("Failed to find copy cache index file %s", indexAbstractFile.getUniquePath()); String msg = String.format("Failed to find copy cache index file %s", indexAbstractFile.getUniquePath());
logger.log(Level.SEVERE, msg); logger.log(Level.SEVERE, msg);
return; return;
} }
// load the data files. We do this now to load them into the cache
for (int i = 0; i < 4; i ++) { for (int i = 0; i < 4; i ++) {
Optional<CacheFileCopy> dataFile = findAndCopyCacheFile(String.format("data_%1d",i), cachePath ); Optional<CacheFileCopy> dataFile = findAndCopyCacheFile(String.format("data_%1d",i), cachePath );
if (!dataFile.isPresent()) { if (!dataFile.isPresent()) {
@ -297,7 +300,7 @@ final class ChromeCacheExtractor {
} }
} }
// find all f_* files in a single query. // find all f_* files in a single query and load them into the cache
findExternalFiles(cachePath); findExternalFiles(cachePath);
} catch (TskCoreException | IngestModuleException ex) { } catch (TskCoreException | IngestModuleException ex) {
@ -306,13 +309,15 @@ final class ChromeCacheExtractor {
return; return;
} }
// parse the index file
logger.log(Level.INFO, "{0}- Now reading Cache index file from path {1}", new Object[]{moduleName, cachePath }); //NON-NLS logger.log(Level.INFO, "{0}- Now reading Cache index file from path {1}", new Object[]{moduleName, cachePath }); //NON-NLS
List<AbstractFile> derivedFiles = new ArrayList<>(); List<AbstractFile> derivedFiles = new ArrayList<>();
Collection<BlackboardArtifact> sourceArtifacts = new ArrayList<>(); Collection<BlackboardArtifact> sourceArtifacts = new ArrayList<>();
Collection<BlackboardArtifact> webCacheArtifacts = new ArrayList<>(); Collection<BlackboardArtifact> webCacheArtifacts = new ArrayList<>();
ByteBuffer indexFileROBuffer = indexFile.get().getByteBuffer(); ByteBuffer indexFileROBuffer = indexFileCopy.get().getByteBuffer();
IndexFileHeader indexHdr = new IndexFileHeader(indexFileROBuffer); IndexFileHeader indexHdr = new IndexFileHeader(indexFileROBuffer);
// seek past the header // seek past the header
@ -326,7 +331,7 @@ final class ChromeCacheExtractor {
"ChromeCacheExtractor.progressMsg", "ChromeCacheExtractor.progressMsg",
moduleName, i, indexHdr.getTableLen(), cachePath) ); moduleName, i, indexHdr.getTableLen(), cachePath) );
try { try {
List<DerivedFile> addedFiles = this.getCacheEntry(addr, sourceArtifacts, webCacheArtifacts); List<DerivedFile> addedFiles = this.processCacheEntry(addr, sourceArtifacts, webCacheArtifacts);
derivedFiles.addAll(addedFiles); derivedFiles.addAll(addedFiles);
} }
catch (TskCoreException | IngestModuleException ex) { catch (TskCoreException | IngestModuleException ex) {
@ -358,15 +363,16 @@ final class ChromeCacheExtractor {
* *
* @return Optional derived file, is a derived file is added for the given entry * @return Optional derived file, is a derived file is added for the given entry
*/ */
List<DerivedFile> getCacheEntry(CacheAddress cacheEntryAddress, Collection<BlackboardArtifact> sourceArtifacts, Collection<BlackboardArtifact> webCacheArtifacts ) throws TskCoreException, IngestModuleException { private List<DerivedFile> processCacheEntry(CacheAddress cacheEntryAddress, Collection<BlackboardArtifact> sourceArtifacts, Collection<BlackboardArtifact> webCacheArtifacts ) throws TskCoreException, IngestModuleException {
List<DerivedFile> derivedFiles = new ArrayList<>(); List<DerivedFile> derivedFiles = new ArrayList<>();
String cacheEntryFileName = cacheEntryAddress.getFilename(); // get the path to the corresponding data_X file
String dataFileName = cacheEntryAddress.getFilename();
String cachePath = cacheEntryAddress.getCachePath(); String cachePath = cacheEntryAddress.getCachePath();
Optional<CacheFileCopy> cacheEntryFile = this.getCacheFileCopy(cacheEntryFileName, cachePath); Optional<CacheFileCopy> cacheEntryFile = this.getCacheFileCopy(dataFileName, cachePath);
if (!cacheEntryFile.isPresent()) { if (!cacheEntryFile.isPresent()) {
String msg = String.format("Failed to get cache entry at address %s", cacheEntryAddress); //NON-NLS String msg = String.format("Failed to get cache entry at address %s", cacheEntryAddress); //NON-NLS
throw new IngestModuleException(msg); throw new IngestModuleException(msg);
@ -375,33 +381,40 @@ final class ChromeCacheExtractor {
// Get the cache entry and its data segments // Get the cache entry and its data segments
CacheEntry cacheEntry = new CacheEntry(cacheEntryAddress, cacheEntryFile.get() ); CacheEntry cacheEntry = new CacheEntry(cacheEntryAddress, cacheEntryFile.get() );
List<CacheData> dataEntries = cacheEntry.getData();
List<CacheData> dataEntries = cacheEntry.getData();
// Only process the first payload data segment in each entry
// first data segement has the HTTP headers, 2nd is the payload
if (dataEntries.size() < 2) {
return derivedFiles;
}
CacheData dataSegment = dataEntries.get(1);
// name of the file that was downloaded and cached (or data_X if it was saved into there)
String cachedFileName = dataSegment.getAddress().getFilename();
Optional<AbstractFile> cachedFileAbstractFile = this.findCacheFile(cachedFileName, cachePath);
if (!cachedFileAbstractFile.isPresent()) {
logger.log(Level.SEVERE, "Error finding file: " + cachePath + "/" + cachedFileName); //NON-NLS
return derivedFiles;
}
boolean isBrotliCompressed = false;
if (dataSegment.getType() != CacheDataTypeEnum.HTTP_HEADER && cacheEntry.isBrotliCompressed() ) {
isBrotliCompressed = true;
}
// setup some attributes for later use
BlackboardAttribute urlAttr = new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_URL, BlackboardAttribute urlAttr = new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_URL,
moduleName, moduleName,
((cacheEntry.getKey() != null) ? cacheEntry.getKey() : "")); ((cacheEntry.getKey() != null) ? cacheEntry.getKey() : ""));
BlackboardAttribute createTimeAttr = new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME_CREATED, BlackboardAttribute createTimeAttr = new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME_CREATED,
moduleName, moduleName,
cacheEntry.getCreationTime()); cacheEntry.getCreationTime());
BlackboardAttribute httpHeaderAttr = new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_HEADERS,
BlackboardAttribute hhtpHeaderAttr = new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_HEADERS,
moduleName, moduleName,
cacheEntry.getHTTPHeaders()); cacheEntry.getHTTPHeaders());
// Only process the first payload data segment in each entry
// first data segement has the HTTP headers, 2nd is the payload
for (int j = 1; j < dataEntries.size() && j < 2; j++) {
CacheData data = dataEntries.get(j);
String dataFilename = data.getAddress().getFilename();
Optional<AbstractFile> dataFile = this.findCacheFile(dataFilename, cachePath);
boolean isBrotliCompressed = false;
if (data.getType() != CacheDataTypeEnum.HTTP_HEADER && cacheEntry.isBrotliCompressed() ) {
isBrotliCompressed = true;
}
Collection<BlackboardAttribute> sourceArtifactAttributes = new ArrayList<>(); Collection<BlackboardAttribute> sourceArtifactAttributes = new ArrayList<>();
sourceArtifactAttributes.add(urlAttr); sourceArtifactAttributes.add(urlAttr);
sourceArtifactAttributes.add(createTimeAttr); sourceArtifactAttributes.add(createTimeAttr);
@ -409,12 +422,13 @@ final class ChromeCacheExtractor {
Collection<BlackboardAttribute> webCacheAttributes = new ArrayList<>(); Collection<BlackboardAttribute> webCacheAttributes = new ArrayList<>();
webCacheAttributes.add(urlAttr); webCacheAttributes.add(urlAttr);
webCacheAttributes.add(createTimeAttr); webCacheAttributes.add(createTimeAttr);
webCacheAttributes.add(hhtpHeaderAttr); webCacheAttributes.add(httpHeaderAttr);
if (dataFile.isPresent()) {
if (data.isInExternalFile() ) { // add artifacts to the f_XXX file
if (dataSegment.isInExternalFile() ) {
try { try {
BlackboardArtifact sourceArtifact = dataFile.get().newArtifact(ARTIFACT_TYPE.TSK_DOWNLOAD_SOURCE); BlackboardArtifact sourceArtifact = cachedFileAbstractFile.get().newArtifact(ARTIFACT_TYPE.TSK_DOWNLOAD_SOURCE);
if (sourceArtifact != null) { if (sourceArtifact != null) {
sourceArtifact.addAttributes(sourceArtifactAttributes); sourceArtifact.addAttributes(sourceArtifactAttributes);
sourceArtifacts.add(sourceArtifact); sourceArtifacts.add(sourceArtifact);
@ -427,32 +441,34 @@ final class ChromeCacheExtractor {
// Add path of f_* file as attribute // Add path of f_* file as attribute
webCacheArtifact.addAttribute(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PATH, webCacheArtifact.addAttribute(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PATH,
moduleName, moduleName,
dataFile.get().getUniquePath())); cachedFileAbstractFile.get().getUniquePath()));
webCacheArtifact.addAttribute(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PATH_ID, webCacheArtifact.addAttribute(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PATH_ID,
moduleName, dataFile.get().getId())); moduleName, cachedFileAbstractFile.get().getId()));
webCacheArtifacts.add(webCacheArtifact); webCacheArtifacts.add(webCacheArtifact);
} }
if (isBrotliCompressed) { if (isBrotliCompressed) {
dataFile.get().setMIMEType(BROTLI_MIMETYPE); cachedFileAbstractFile.get().setMIMEType(BROTLI_MIMETYPE);
dataFile.get().save(); cachedFileAbstractFile.get().save();
} }
} catch (TskException ex) { } catch (TskException ex) {
logger.log(Level.SEVERE, "Error while trying to add an artifact", ex); //NON-NLS logger.log(Level.SEVERE, "Error while trying to add an artifact", ex); //NON-NLS
} }
} else { }
// extract the embedded data to a derived file and create artifacts
else {
// Data segments in "data_x" files are saved in individual files and added as derived files // Data segments in "data_x" files are saved in individual files and added as derived files
String filename = data.save(); String filename = dataSegment.save();
String relPathname = getRelOutputFolderName() + data.getAddress().getCachePath() + filename; String relPathname = getRelOutputFolderName() + dataSegment.getAddress().getCachePath() + filename;
try { try {
DerivedFile derivedFile = fileManager.addDerivedFile(filename, relPathname, DerivedFile derivedFile = fileManager.addDerivedFile(filename, relPathname,
data.getDataLength(), dataSegment.getDataLength(),
cacheEntry.getCreationTime(), cacheEntry.getCreationTime(), cacheEntry.getCreationTime(), cacheEntry.getCreationTime(), // TBD cacheEntry.getCreationTime(), cacheEntry.getCreationTime(), cacheEntry.getCreationTime(), cacheEntry.getCreationTime(), // TBD
true, true,
dataFile.get(), cachedFileAbstractFile.get(),
"", "",
moduleName, moduleName,
VERSION_NUMBER, VERSION_NUMBER,
@ -490,8 +506,6 @@ final class ChromeCacheExtractor {
logger.log(Level.SEVERE, "Error while trying to add an artifact", ex); //NON-NLS logger.log(Level.SEVERE, "Error while trying to add an artifact", ex); //NON-NLS
} }
} }
}
}
return derivedFiles; return derivedFiles;
} }
@ -519,20 +533,22 @@ final class ChromeCacheExtractor {
* @return Optional abstract file * @return Optional abstract file
* @throws TskCoreException * @throws TskCoreException
*/ */
Optional<AbstractFile> findCacheFile(String cacheFileName, String cachePath) throws TskCoreException { private Optional<AbstractFile> findCacheFile(String cacheFileName, String cachePath) throws TskCoreException {
// see if it is cached
String fileTableKey = cachePath + cacheFileName; String fileTableKey = cachePath + cacheFileName;
if (cacheFileName.startsWith("f_") && externalFilesTable.containsKey(fileTableKey)) { if (cacheFileName.startsWith("f_") && externalFilesTable.containsKey(fileTableKey)) {
return Optional.of(externalFilesTable.get(fileTableKey)); return Optional.of(externalFilesTable.get(fileTableKey));
} }
if (filesTable.containsKey(fileTableKey)) { if (fileCopyCache.containsKey(fileTableKey)) {
return Optional.of(filesTable.get(fileTableKey).getAbstractFile()); return Optional.of(fileCopyCache.get(fileTableKey).getAbstractFile());
} }
List<AbstractFile> cacheFiles = fileManager.findFiles(dataSource, cacheFileName, cachePath); //NON-NLS List<AbstractFile> cacheFiles = fileManager.findFiles(dataSource, cacheFileName, cachePath); //NON-NLS
if (!cacheFiles.isEmpty()) { if (!cacheFiles.isEmpty()) {
for (AbstractFile abstractFile: cacheFiles ) { for (AbstractFile abstractFile: cacheFiles ) {
if (abstractFile.getUniquePath().trim().endsWith(DEFAULT_CACHE_STR)) { if (abstractFile.getUniquePath().trim().endsWith(DEFAULT_CACHE_PATH_STR)) {
return Optional.of(abstractFile); return Optional.of(abstractFile);
} }
} }
@ -545,12 +561,11 @@ final class ChromeCacheExtractor {
/** /**
* Finds abstract file(s) for a cache file with the specified name. * Finds abstract file(s) for a cache file with the specified name.
* *
* @param cacheFileName
* @return list of abstract files matching the specified file name * @return list of abstract files matching the specified file name
* @throws TskCoreException * @throws TskCoreException
*/ */
List<AbstractFile> findCacheFiles(String cacheFileName) throws TskCoreException { private List<AbstractFile> findCacheIndexFiles() throws TskCoreException {
return fileManager.findFiles(dataSource, cacheFileName, DEFAULT_CACHE_STR); //NON-NLS return fileManager.findFiles(dataSource, "index", DEFAULT_CACHE_PATH_STR); //NON-NLS
} }
@ -558,16 +573,17 @@ final class ChromeCacheExtractor {
* Returns CacheFileCopy for the specified file from the file table. * Returns CacheFileCopy for the specified file from the file table.
* Find the file and creates a copy if it isn't already in the table. * Find the file and creates a copy if it isn't already in the table.
* *
* @param cacheFileName * @param cacheFileName Name of file
* @param cachePath Parent path of file
* @return CacheFileCopy * @return CacheFileCopy
* @throws TskCoreException * @throws TskCoreException
*/ */
Optional<CacheFileCopy> getCacheFileCopy(String cacheFileName, String cachePath) throws TskCoreException, IngestModuleException { private Optional<CacheFileCopy> getCacheFileCopy(String cacheFileName, String cachePath) throws TskCoreException, IngestModuleException {
// Check if the file is already in the table // Check if the file is already in the cache
String fileTableKey = cachePath + cacheFileName; String fileTableKey = cachePath + cacheFileName;
if (filesTable.containsKey(fileTableKey)) { if (fileCopyCache.containsKey(fileTableKey)) {
return Optional.of(filesTable.get(fileTableKey)); return Optional.of(fileCopyCache.get(fileTableKey));
} }
return findAndCopyCacheFile(cacheFileName, cachePath); return findAndCopyCacheFile(cacheFileName, cachePath);
@ -580,13 +596,17 @@ final class ChromeCacheExtractor {
* @return Cache file copy * @return Cache file copy
* @throws TskCoreException * @throws TskCoreException
*/ */
Optional<CacheFileCopy> findAndCopyCacheFile(String cacheFileName, String cachePath) throws TskCoreException, IngestModuleException { private Optional<CacheFileCopy> findAndCopyCacheFile(String cacheFileName, String cachePath) throws TskCoreException, IngestModuleException {
Optional<AbstractFile> cacheFileOptional = findCacheFile(cacheFileName, cachePath); Optional<AbstractFile> cacheFileOptional = findCacheFile(cacheFileName, cachePath);
if (!cacheFileOptional.isPresent()) { if (!cacheFileOptional.isPresent()) {
return Optional.empty(); return Optional.empty();
} }
// write the file to disk so that we can have a memory-mapped ByteBuffer
// @@@ NOTE: I"m not sure this is needed. These files are small enough and we could probably just load them into
// a byte[] for ByteBuffer.
AbstractFile cacheFile = cacheFileOptional.get(); AbstractFile cacheFile = cacheFileOptional.get();
RandomAccessFile randomAccessFile = null; RandomAccessFile randomAccessFile = null;
String tempFilePathname = RAImageIngestModule.getRATempPath(currentCase, moduleName) + cachePath + cacheFile.getName(); //NON-NLS String tempFilePathname = RAImageIngestModule.getRATempPath(currentCase, moduleName) + cachePath + cacheFile.getName(); //NON-NLS
@ -603,7 +623,7 @@ final class ChromeCacheExtractor {
CacheFileCopy cacheFileCopy = new CacheFileCopy(cacheFile, randomAccessFile, cacheFileROBuf ); CacheFileCopy cacheFileCopy = new CacheFileCopy(cacheFile, randomAccessFile, cacheFileROBuf );
if (!cacheFileName.startsWith("f_")) { if (!cacheFileName.startsWith("f_")) {
filesTable.put(cachePath + cacheFileName, cacheFileCopy); fileCopyCache.put(cachePath + cacheFileName, cacheFileCopy);
} }
return Optional.of(cacheFileCopy); return Optional.of(cacheFileCopy);
@ -787,6 +807,10 @@ final class ChromeCacheExtractor {
return fileType; return fileType;
} }
/**
* Name where cached file is stored. Either a data or f_ file.
* @return
*/
String getFilename() { String getFilename() {
return fileName; return fileName;
} }