diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ChromeCacheExtractor.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ChromeCacheExtractor.java index 7cf801efb1..ce7c1c44ee 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ChromeCacheExtractor.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ChromeCacheExtractor.java @@ -75,7 +75,7 @@ import org.sleuthkit.datamodel.TskException; */ final class ChromeCacheExtractor { - private final static String DEFAULT_CACHE_STR = "default/cache"; //NON-NLS + private final static String DEFAULT_CACHE_PATH_STR = "default/cache"; //NON-NLS private final static String BROTLI_MIMETYPE ="application/x-brotli"; //NON-NLS private final static long UINT32_MASK = 0xFFFFFFFFl; @@ -99,7 +99,7 @@ final class ChromeCacheExtractor { private FileManager fileManager; // A file table to cache copies of index and data_n files. - private final Map filesTable = new HashMap<>(); + private final Map fileCopyCache = new HashMap<>(); // A file table to cache the f_* files. private final Map externalFilesTable = new HashMap<>(); @@ -152,7 +152,7 @@ final class ChromeCacheExtractor { * * @throws IngestModuleException */ - void moduleInit() throws IngestModuleException { + private void moduleInit() throws IngestModuleException { try { currentCase = Case.getCurrentCaseThrows(); @@ -179,9 +179,9 @@ final class ChromeCacheExtractor { * * @throws org.sleuthkit.autopsy.ingest.IngestModule.IngestModuleException */ - void subInit(String cachePath) throws IngestModuleException { + private void resetForNewFolder(String cachePath) throws IngestModuleException { - filesTable.clear(); + fileCopyCache.clear(); externalFilesTable.clear(); String cacheAbsOutputFolderName = this.getAbsOutputFolderName() + cachePath; @@ -203,9 +203,9 @@ final class ChromeCacheExtractor { * Removes any temp copies of cache files created during extraction. * */ - void cleanup () { + private void cleanup () { - for (Entry entry : this.filesTable.entrySet()) { + for (Entry entry : this.fileCopyCache.entrySet()) { Path tempFilePath = Paths.get(RAImageIngestModule.getRATempPath(currentCase, moduleName), entry.getKey() ); try { entry.getValue().getFileCopy().getChannel().close(); @@ -240,7 +240,7 @@ final class ChromeCacheExtractor { } /** - * Extracts the data from Chrome caches + * Extracts the data from Chrome caches . Main entry point for the analysis * * A data source may have multiple Chrome user profiles and caches. * @@ -255,14 +255,14 @@ final class ChromeCacheExtractor { return; } - // Find all possible caches + // Find and process the cache folders. There could be one per user List indexFiles; try { - indexFiles = findCacheFiles("index"); //NON-NLS + indexFiles = findCacheIndexFiles(); - // Get each of the caches + // Process each of the caches for (AbstractFile indexFile: indexFiles) { - getCache(indexFile); + processCacheIndexFile(indexFile); } } catch (TskCoreException ex) { @@ -272,24 +272,27 @@ final class ChromeCacheExtractor { } /** - * Extracts the cache for the specified cache index file. + * Processes a user's cache and creates corresponding artifacts and derived files. * - * @param cacheIndexFile + * @param cacheIndexFile Cache index file for a given user */ - void getCache(AbstractFile indexAbstractFile) { + private void processCacheIndexFile(AbstractFile indexAbstractFile) { String cachePath = indexAbstractFile.getParentPath(); - Optional indexFile; + Optional indexFileCopy; try { - subInit(cachePath); + resetForNewFolder(cachePath); - indexFile = this.getCacheFileCopy(indexAbstractFile.getName(), cachePath); - if (!indexFile.isPresent()) { + // @@@ This is little ineffecient because we later in this call search for the AbstractFile that we currently have + indexFileCopy = this.getCacheFileCopy(indexAbstractFile.getName(), cachePath); + if (!indexFileCopy.isPresent()) { String msg = String.format("Failed to find copy cache index file %s", indexAbstractFile.getUniquePath()); logger.log(Level.SEVERE, msg); return; } + + // load the data files. We do this now to load them into the cache for (int i = 0; i < 4; i ++) { Optional dataFile = findAndCopyCacheFile(String.format("data_%1d",i), cachePath ); if (!dataFile.isPresent()) { @@ -297,7 +300,7 @@ final class ChromeCacheExtractor { } } - // find all f_* files in a single query. + // find all f_* files in a single query and load them into the cache findExternalFiles(cachePath); } catch (TskCoreException | IngestModuleException ex) { @@ -306,13 +309,15 @@ final class ChromeCacheExtractor { return; } + + // parse the index file logger.log(Level.INFO, "{0}- Now reading Cache index file from path {1}", new Object[]{moduleName, cachePath }); //NON-NLS List derivedFiles = new ArrayList<>(); Collection sourceArtifacts = new ArrayList<>(); Collection webCacheArtifacts = new ArrayList<>(); - ByteBuffer indexFileROBuffer = indexFile.get().getByteBuffer(); + ByteBuffer indexFileROBuffer = indexFileCopy.get().getByteBuffer(); IndexFileHeader indexHdr = new IndexFileHeader(indexFileROBuffer); // seek past the header @@ -326,7 +331,7 @@ final class ChromeCacheExtractor { "ChromeCacheExtractor.progressMsg", moduleName, i, indexHdr.getTableLen(), cachePath) ); try { - List addedFiles = this.getCacheEntry(addr, sourceArtifacts, webCacheArtifacts); + List addedFiles = this.processCacheEntry(addr, sourceArtifacts, webCacheArtifacts); derivedFiles.addAll(addedFiles); } catch (TskCoreException | IngestModuleException ex) { @@ -358,15 +363,16 @@ final class ChromeCacheExtractor { * * @return Optional derived file, is a derived file is added for the given entry */ - List getCacheEntry(CacheAddress cacheEntryAddress, Collection sourceArtifacts, Collection webCacheArtifacts ) throws TskCoreException, IngestModuleException { + private List processCacheEntry(CacheAddress cacheEntryAddress, Collection sourceArtifacts, Collection webCacheArtifacts ) throws TskCoreException, IngestModuleException { List derivedFiles = new ArrayList<>(); - String cacheEntryFileName = cacheEntryAddress.getFilename(); + // get the path to the corresponding data_X file + String dataFileName = cacheEntryAddress.getFilename(); String cachePath = cacheEntryAddress.getCachePath(); - Optional cacheEntryFile = this.getCacheFileCopy(cacheEntryFileName, cachePath); + Optional cacheEntryFile = this.getCacheFileCopy(dataFileName, cachePath); if (!cacheEntryFile.isPresent()) { String msg = String.format("Failed to get cache entry at address %s", cacheEntryAddress); //NON-NLS throw new IngestModuleException(msg); @@ -375,121 +381,129 @@ final class ChromeCacheExtractor { // Get the cache entry and its data segments CacheEntry cacheEntry = new CacheEntry(cacheEntryAddress, cacheEntryFile.get() ); + List dataEntries = cacheEntry.getData(); + // Only process the first payload data segment in each entry + // first data segement has the HTTP headers, 2nd is the payload + if (dataEntries.size() < 2) { + return derivedFiles; + } + CacheData dataSegment = dataEntries.get(1); + + // name of the file that was downloaded and cached (or data_X if it was saved into there) + String cachedFileName = dataSegment.getAddress().getFilename(); + Optional cachedFileAbstractFile = this.findCacheFile(cachedFileName, cachePath); + if (!cachedFileAbstractFile.isPresent()) { + logger.log(Level.SEVERE, "Error finding file: " + cachePath + "/" + cachedFileName); //NON-NLS + return derivedFiles; + } + + boolean isBrotliCompressed = false; + if (dataSegment.getType() != CacheDataTypeEnum.HTTP_HEADER && cacheEntry.isBrotliCompressed() ) { + isBrotliCompressed = true; + } + + // setup some attributes for later use BlackboardAttribute urlAttr = new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_URL, moduleName, ((cacheEntry.getKey() != null) ? cacheEntry.getKey() : "")); - BlackboardAttribute createTimeAttr = new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME_CREATED, moduleName, cacheEntry.getCreationTime()); - - BlackboardAttribute hhtpHeaderAttr = new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_HEADERS, + BlackboardAttribute httpHeaderAttr = new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_HEADERS, moduleName, cacheEntry.getHTTPHeaders()); - - // Only process the first payload data segment in each entry - // first data segement has the HTTP headers, 2nd is the payload - for (int j = 1; j < dataEntries.size() && j < 2; j++) { - CacheData data = dataEntries.get(j); - String dataFilename = data.getAddress().getFilename(); - Optional dataFile = this.findCacheFile(dataFilename, cachePath); + Collection sourceArtifactAttributes = new ArrayList<>(); + sourceArtifactAttributes.add(urlAttr); + sourceArtifactAttributes.add(createTimeAttr); - boolean isBrotliCompressed = false; - if (data.getType() != CacheDataTypeEnum.HTTP_HEADER && cacheEntry.isBrotliCompressed() ) { - isBrotliCompressed = true; - } + Collection webCacheAttributes = new ArrayList<>(); + webCacheAttributes.add(urlAttr); + webCacheAttributes.add(createTimeAttr); + webCacheAttributes.add(httpHeaderAttr); - Collection sourceArtifactAttributes = new ArrayList<>(); - sourceArtifactAttributes.add(urlAttr); - sourceArtifactAttributes.add(createTimeAttr); - - Collection webCacheAttributes = new ArrayList<>(); - webCacheAttributes.add(urlAttr); - webCacheAttributes.add(createTimeAttr); - webCacheAttributes.add(hhtpHeaderAttr); - - if (dataFile.isPresent()) { - if (data.isInExternalFile() ) { - try { - BlackboardArtifact sourceArtifact = dataFile.get().newArtifact(ARTIFACT_TYPE.TSK_DOWNLOAD_SOURCE); - if (sourceArtifact != null) { - sourceArtifact.addAttributes(sourceArtifactAttributes); - sourceArtifacts.add(sourceArtifact); - } - - BlackboardArtifact webCacheArtifact = cacheEntryFile.get().getAbstractFile().newArtifact(ARTIFACT_TYPE.TSK_WEB_CACHE); - if (webCacheArtifact != null) { - webCacheArtifact.addAttributes(webCacheAttributes); - - // Add path of f_* file as attribute - webCacheArtifact.addAttribute(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PATH, - moduleName, - dataFile.get().getUniquePath())); - - webCacheArtifact.addAttribute(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PATH_ID, - moduleName, dataFile.get().getId())); - - webCacheArtifacts.add(webCacheArtifact); - } - - if (isBrotliCompressed) { - dataFile.get().setMIMEType(BROTLI_MIMETYPE); - dataFile.get().save(); - } - } catch (TskException ex) { - logger.log(Level.SEVERE, "Error while trying to add an artifact", ex); //NON-NLS - } - } else { - - // Data segments in "data_x" files are saved in individual files and added as derived files - String filename = data.save(); - String relPathname = getRelOutputFolderName() + data.getAddress().getCachePath() + filename; - try { - DerivedFile derivedFile = fileManager.addDerivedFile(filename, relPathname, - data.getDataLength(), - cacheEntry.getCreationTime(), cacheEntry.getCreationTime(), cacheEntry.getCreationTime(), cacheEntry.getCreationTime(), // TBD - true, - dataFile.get(), - "", - moduleName, - VERSION_NUMBER, - "", - TskData.EncodingType.NONE); - - BlackboardArtifact sourceArtifact = derivedFile.newArtifact(ARTIFACT_TYPE.TSK_DOWNLOAD_SOURCE); - if (sourceArtifact != null) { - sourceArtifact.addAttributes(sourceArtifactAttributes); - sourceArtifacts.add(sourceArtifact); - } - - BlackboardArtifact webCacheArtifact = cacheEntryFile.get().getAbstractFile().newArtifact(ARTIFACT_TYPE.TSK_WEB_CACHE); - if (webCacheArtifact != null) { - webCacheArtifact.addAttributes(webCacheAttributes); - - // Add path of derived file as attribute - webCacheArtifact.addAttribute(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PATH, - moduleName, - derivedFile.getUniquePath())); - - webCacheArtifact.addAttribute(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PATH_ID, - moduleName, derivedFile.getId())); - - webCacheArtifacts.add(webCacheArtifact); - } - - if (isBrotliCompressed) { - derivedFile.setMIMEType(BROTLI_MIMETYPE); - derivedFile.save(); - } - - derivedFiles.add(derivedFile); - } catch (TskException ex) { - logger.log(Level.SEVERE, "Error while trying to add an artifact", ex); //NON-NLS - } + + // add artifacts to the f_XXX file + if (dataSegment.isInExternalFile() ) { + try { + BlackboardArtifact sourceArtifact = cachedFileAbstractFile.get().newArtifact(ARTIFACT_TYPE.TSK_DOWNLOAD_SOURCE); + if (sourceArtifact != null) { + sourceArtifact.addAttributes(sourceArtifactAttributes); + sourceArtifacts.add(sourceArtifact); } + + BlackboardArtifact webCacheArtifact = cacheEntryFile.get().getAbstractFile().newArtifact(ARTIFACT_TYPE.TSK_WEB_CACHE); + if (webCacheArtifact != null) { + webCacheArtifact.addAttributes(webCacheAttributes); + + // Add path of f_* file as attribute + webCacheArtifact.addAttribute(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PATH, + moduleName, + cachedFileAbstractFile.get().getUniquePath())); + + webCacheArtifact.addAttribute(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PATH_ID, + moduleName, cachedFileAbstractFile.get().getId())); + + webCacheArtifacts.add(webCacheArtifact); + } + + if (isBrotliCompressed) { + cachedFileAbstractFile.get().setMIMEType(BROTLI_MIMETYPE); + cachedFileAbstractFile.get().save(); + } + } catch (TskException ex) { + logger.log(Level.SEVERE, "Error while trying to add an artifact", ex); //NON-NLS + } + } + // extract the embedded data to a derived file and create artifacts + else { + + // Data segments in "data_x" files are saved in individual files and added as derived files + String filename = dataSegment.save(); + String relPathname = getRelOutputFolderName() + dataSegment.getAddress().getCachePath() + filename; + try { + DerivedFile derivedFile = fileManager.addDerivedFile(filename, relPathname, + dataSegment.getDataLength(), + cacheEntry.getCreationTime(), cacheEntry.getCreationTime(), cacheEntry.getCreationTime(), cacheEntry.getCreationTime(), // TBD + true, + cachedFileAbstractFile.get(), + "", + moduleName, + VERSION_NUMBER, + "", + TskData.EncodingType.NONE); + + BlackboardArtifact sourceArtifact = derivedFile.newArtifact(ARTIFACT_TYPE.TSK_DOWNLOAD_SOURCE); + if (sourceArtifact != null) { + sourceArtifact.addAttributes(sourceArtifactAttributes); + sourceArtifacts.add(sourceArtifact); + } + + BlackboardArtifact webCacheArtifact = cacheEntryFile.get().getAbstractFile().newArtifact(ARTIFACT_TYPE.TSK_WEB_CACHE); + if (webCacheArtifact != null) { + webCacheArtifact.addAttributes(webCacheAttributes); + + // Add path of derived file as attribute + webCacheArtifact.addAttribute(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PATH, + moduleName, + derivedFile.getUniquePath())); + + webCacheArtifact.addAttribute(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PATH_ID, + moduleName, derivedFile.getId())); + + webCacheArtifacts.add(webCacheArtifact); + } + + if (isBrotliCompressed) { + derivedFile.setMIMEType(BROTLI_MIMETYPE); + derivedFile.save(); + } + + derivedFiles.add(derivedFile); + } catch (TskException ex) { + logger.log(Level.SEVERE, "Error while trying to add an artifact", ex); //NON-NLS } } @@ -519,20 +533,22 @@ final class ChromeCacheExtractor { * @return Optional abstract file * @throws TskCoreException */ - Optional findCacheFile(String cacheFileName, String cachePath) throws TskCoreException { + private Optional findCacheFile(String cacheFileName, String cachePath) throws TskCoreException { + // see if it is cached String fileTableKey = cachePath + cacheFileName; if (cacheFileName.startsWith("f_") && externalFilesTable.containsKey(fileTableKey)) { return Optional.of(externalFilesTable.get(fileTableKey)); } - if (filesTable.containsKey(fileTableKey)) { - return Optional.of(filesTable.get(fileTableKey).getAbstractFile()); + if (fileCopyCache.containsKey(fileTableKey)) { + return Optional.of(fileCopyCache.get(fileTableKey).getAbstractFile()); } + List cacheFiles = fileManager.findFiles(dataSource, cacheFileName, cachePath); //NON-NLS if (!cacheFiles.isEmpty()) { for (AbstractFile abstractFile: cacheFiles ) { - if (abstractFile.getUniquePath().trim().endsWith(DEFAULT_CACHE_STR)) { + if (abstractFile.getUniquePath().trim().endsWith(DEFAULT_CACHE_PATH_STR)) { return Optional.of(abstractFile); } } @@ -545,29 +561,29 @@ final class ChromeCacheExtractor { /** * Finds abstract file(s) for a cache file with the specified name. * - * @param cacheFileName * @return list of abstract files matching the specified file name * @throws TskCoreException */ - List findCacheFiles(String cacheFileName) throws TskCoreException { - return fileManager.findFiles(dataSource, cacheFileName, DEFAULT_CACHE_STR); //NON-NLS + private List findCacheIndexFiles() throws TskCoreException { + return fileManager.findFiles(dataSource, "index", DEFAULT_CACHE_PATH_STR); //NON-NLS } /** * Returns CacheFileCopy for the specified file from the file table. * Find the file and creates a copy if it isn't already in the table. - * - * @param cacheFileName + * + * @param cacheFileName Name of file + * @param cachePath Parent path of file * @return CacheFileCopy * @throws TskCoreException */ - Optional getCacheFileCopy(String cacheFileName, String cachePath) throws TskCoreException, IngestModuleException { + private Optional getCacheFileCopy(String cacheFileName, String cachePath) throws TskCoreException, IngestModuleException { - // Check if the file is already in the table + // Check if the file is already in the cache String fileTableKey = cachePath + cacheFileName; - if (filesTable.containsKey(fileTableKey)) { - return Optional.of(filesTable.get(fileTableKey)); + if (fileCopyCache.containsKey(fileTableKey)) { + return Optional.of(fileCopyCache.get(fileTableKey)); } return findAndCopyCacheFile(cacheFileName, cachePath); @@ -580,13 +596,17 @@ final class ChromeCacheExtractor { * @return Cache file copy * @throws TskCoreException */ - Optional findAndCopyCacheFile(String cacheFileName, String cachePath) throws TskCoreException, IngestModuleException { + private Optional findAndCopyCacheFile(String cacheFileName, String cachePath) throws TskCoreException, IngestModuleException { Optional cacheFileOptional = findCacheFile(cacheFileName, cachePath); if (!cacheFileOptional.isPresent()) { return Optional.empty(); } + + // write the file to disk so that we can have a memory-mapped ByteBuffer + // @@@ NOTE: I"m not sure this is needed. These files are small enough and we could probably just load them into + // a byte[] for ByteBuffer. AbstractFile cacheFile = cacheFileOptional.get(); RandomAccessFile randomAccessFile = null; String tempFilePathname = RAImageIngestModule.getRATempPath(currentCase, moduleName) + cachePath + cacheFile.getName(); //NON-NLS @@ -603,7 +623,7 @@ final class ChromeCacheExtractor { CacheFileCopy cacheFileCopy = new CacheFileCopy(cacheFile, randomAccessFile, cacheFileROBuf ); if (!cacheFileName.startsWith("f_")) { - filesTable.put(cachePath + cacheFileName, cacheFileCopy); + fileCopyCache.put(cachePath + cacheFileName, cacheFileCopy); } return Optional.of(cacheFileCopy); @@ -787,6 +807,10 @@ final class ChromeCacheExtractor { return fileType; } + /** + * Name where cached file is stored. Either a data or f_ file. + * @return + */ String getFilename() { return fileName; }