mirror of
https://github.com/overcuriousity/autopsy-flatpak.git
synced 2025-07-13 00:16:16 +00:00
1060 lines
39 KiB
Java
1060 lines
39 KiB
Java
/*
|
|
*
|
|
* Autopsy Forensic Browser
|
|
*
|
|
* Copyright 2019 Basis Technology Corp.
|
|
*
|
|
* Project Contact/Architect: carrier <at> sleuthkit <dot> org
|
|
*
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
* you may not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
* See the License for the specific language governing permissions and
|
|
* limitations under the License.
|
|
*/
|
|
package org.sleuthkit.autopsy.recentactivity;
|
|
|
|
import java.io.File;
|
|
import java.io.FileOutputStream;
|
|
import java.io.IOException;
|
|
import java.io.RandomAccessFile;
|
|
import java.nio.ByteBuffer;
|
|
import java.nio.ByteOrder;
|
|
import java.nio.channels.FileChannel;
|
|
import java.util.ArrayList;
|
|
import java.util.Collection;
|
|
import java.util.HashMap;
|
|
import java.util.List;
|
|
import java.util.Map;
|
|
import java.util.Map.Entry;
|
|
import java.util.Optional;
|
|
import java.util.logging.Level;
|
|
import org.openide.util.Exceptions;
|
|
import org.openide.util.NbBundle;
|
|
import org.sleuthkit.autopsy.casemodule.Case;
|
|
import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException;
|
|
import org.sleuthkit.autopsy.casemodule.services.FileManager;
|
|
import org.sleuthkit.autopsy.coreutils.Logger;
|
|
import org.sleuthkit.autopsy.datamodel.ContentUtils;
|
|
import org.sleuthkit.autopsy.ingest.IngestJobContext;
|
|
import org.sleuthkit.autopsy.ingest.IngestModule.IngestModuleException;
|
|
import org.sleuthkit.autopsy.ingest.IngestServices;
|
|
import org.sleuthkit.autopsy.ingest.ModuleContentEvent;
|
|
import org.sleuthkit.autopsy.ingest.ModuleDataEvent;
|
|
import org.sleuthkit.autopsy.modules.filetypeid.FileTypeDetector;
|
|
import org.sleuthkit.datamodel.AbstractFile;
|
|
import org.sleuthkit.datamodel.BlackboardArtifact;
|
|
import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE;
|
|
import org.sleuthkit.datamodel.BlackboardAttribute;
|
|
import org.sleuthkit.datamodel.Content;
|
|
import org.sleuthkit.datamodel.DerivedFile;
|
|
import org.sleuthkit.datamodel.ReadContentInputStream;
|
|
import org.sleuthkit.datamodel.SleuthkitCase;
|
|
import org.sleuthkit.datamodel.TimeUtilities;
|
|
import org.sleuthkit.datamodel.TskCoreException;
|
|
import org.sleuthkit.datamodel.TskData;
|
|
import org.sleuthkit.datamodel.TskException;
|
|
|
|
/**
|
|
* Extracts and parses Chrome Cache files.
|
|
*/
|
|
final class ChromeCacheExtractor {
|
|
|
|
private final long UINT32_MASK = 0xFFFFFFFFl;
|
|
|
|
private final int INDEXFILE_HDR_SIZE = 92*4;
|
|
private final int DATAFILE_HDR_SIZE = 8192;
|
|
|
|
private final Logger logger = Logger.getLogger(this.getClass().getName());
|
|
|
|
private static final String VERSION_NUMBER = "1.0.0";
|
|
private final String moduleName;
|
|
|
|
private String absOutputFolderName;
|
|
private String relOutputFolderName;
|
|
|
|
private final Content dataSource;
|
|
private final IngestJobContext context;
|
|
private IngestServices services = IngestServices.getInstance();
|
|
private Case currentCase;
|
|
private FileManager fileManager;
|
|
private FileTypeDetector fileTypeDetector;
|
|
|
|
|
|
private Map<String, CacheFileCopy> filesTable = new HashMap<>();
|
|
|
|
/**
|
|
* Encapsulates abstract file for a cache file as well as a temp file copy
|
|
* that can be accessed as a random access file.
|
|
*/
|
|
final class CacheFileCopy {
|
|
|
|
private AbstractFile abstractFile;
|
|
private RandomAccessFile fileCopy;
|
|
private ByteBuffer byteBuffer;
|
|
|
|
CacheFileCopy (AbstractFile abstractFile, RandomAccessFile fileCopy, ByteBuffer buffer ) {
|
|
this.abstractFile = abstractFile;
|
|
this.fileCopy = fileCopy;
|
|
this.byteBuffer = buffer;
|
|
}
|
|
|
|
public RandomAccessFile getFileCopy() {
|
|
return fileCopy;
|
|
}
|
|
public ByteBuffer getByteBuffer() {
|
|
return byteBuffer;
|
|
}
|
|
AbstractFile getAbstractFile() {
|
|
return abstractFile;
|
|
}
|
|
}
|
|
|
|
ChromeCacheExtractor(Content dataSource, IngestJobContext context ) {
|
|
moduleName = NbBundle.getMessage(ChromeCacheExtractor.class, "ChromeCacheExtractor.moduleName");
|
|
this.dataSource = dataSource;
|
|
this.context = context;
|
|
}
|
|
|
|
|
|
/**
|
|
* Initializes Chrome cache extractor module
|
|
*
|
|
* @throws org.sleuthkit.autopsy.ingest.IngestModule.IngestModuleException
|
|
*/
|
|
void init() throws IngestModuleException {
|
|
|
|
try {
|
|
currentCase = Case.getCurrentCaseThrows();
|
|
fileManager = currentCase.getServices().getFileManager();
|
|
fileTypeDetector = new FileTypeDetector();
|
|
|
|
// Create an output folder to save any derived files
|
|
absOutputFolderName = RAImageIngestModule.getRAOutputPath(currentCase, moduleName);
|
|
relOutputFolderName = RAImageIngestModule.getRelModuleOutputPath() + File.separator + moduleName;
|
|
File dir = new File(absOutputFolderName);
|
|
if (dir.exists() == false) {
|
|
dir.mkdirs();
|
|
}
|
|
} catch (NoCurrentCaseException ex) {
|
|
String msg = "Failed to get current case.";
|
|
throw new IngestModuleException(msg, ex);
|
|
} catch (FileTypeDetector.FileTypeDetectorInitException ex) {
|
|
String msg = "Failed to get FileTypeDetector.";
|
|
throw new IngestModuleException(msg, ex);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Cleans up after the module is done
|
|
*
|
|
*/
|
|
void cleanup () {
|
|
|
|
for (Entry<String, CacheFileCopy> entry : this.filesTable.entrySet()) {
|
|
String tempFilePathname = RAImageIngestModule.getRATempPath(currentCase, moduleName) + File.separator + entry.getKey();
|
|
try {
|
|
entry.getValue().getFileCopy().getChannel().close();
|
|
entry.getValue().getFileCopy().close();
|
|
|
|
File tmpFile = new File(tempFilePathname);
|
|
if (!tmpFile.delete()) {
|
|
tmpFile.deleteOnExit();
|
|
}
|
|
} catch (IOException ex) {
|
|
logger.log(Level.SEVERE, String.format("Failed to delete cache file copy %s", tempFilePathname), ex);
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Returns the location of output folder for this module
|
|
*
|
|
* @return
|
|
*/
|
|
private String getAbsOutputFolderName() {
|
|
return absOutputFolderName;
|
|
}
|
|
|
|
/**
|
|
* Returns the relative location of output folder for this module
|
|
*
|
|
* @return
|
|
*/
|
|
private String getRelOutputFolderName() {
|
|
return relOutputFolderName;
|
|
}
|
|
|
|
/**
|
|
* Extracts the data from Chrome cache
|
|
*/
|
|
void getCache() {
|
|
|
|
try {
|
|
init();
|
|
} catch (IngestModuleException ex) {
|
|
|
|
// TBD: show the error to Autospy error console??
|
|
String msg = "Failed to initialize ChromeCacheExtractor.";
|
|
logger.log(Level.SEVERE, msg, ex);
|
|
return;
|
|
}
|
|
|
|
Optional<CacheFileCopy> indexFile;
|
|
try {
|
|
// find the index file
|
|
indexFile = findAndCopyCacheFile("index");
|
|
if (!indexFile.isPresent()) {
|
|
return;
|
|
}
|
|
|
|
for (int i = 0; i < 4; i ++) {
|
|
Optional<CacheFileCopy> dataFile = findAndCopyCacheFile(String.format("data_%1d",i));
|
|
if (!dataFile.isPresent()) {
|
|
return;
|
|
}
|
|
}
|
|
|
|
} catch (TskCoreException | IngestModuleException ex) {
|
|
String msg = "Failed to find cache files";
|
|
logger.log(Level.SEVERE, msg, ex);
|
|
return;
|
|
}
|
|
|
|
logger.log(Level.INFO, "{0}- Now reading Cache index file", new Object[]{moduleName}); //NON-NLS
|
|
|
|
List<AbstractFile> derivedFiles = new ArrayList<>();
|
|
|
|
ByteBuffer indexFileROBuffer = indexFile.get().getByteBuffer();
|
|
IndexFileHeader indexHdr = new IndexFileHeader(indexFileROBuffer);
|
|
|
|
// seek past the header
|
|
indexFileROBuffer.position(INDEXFILE_HDR_SIZE);
|
|
|
|
// Process each address in the table
|
|
for (int i = 0; i < indexHdr.getTableLen(); i++) {
|
|
CacheAddress addr = new CacheAddress(indexFileROBuffer.getInt() & UINT32_MASK);
|
|
|
|
if (addr.isInitialized()) {
|
|
String fileName = addr.getFilename();
|
|
try {
|
|
Optional<CacheFileCopy> cacheFileCopy = this.getCacheFileCopy(fileName);
|
|
if (!cacheFileCopy.isPresent()) {
|
|
logger.log(Level.SEVERE, String.format("Failed to get cache entry at address %s", addr)); //NON-NLS
|
|
}
|
|
|
|
// Get the cache entry at this address
|
|
CacheEntry cacheEntry = new CacheEntry(addr, cacheFileCopy.get() );
|
|
|
|
// Get the data segments - each entry can have up to 4 data segments
|
|
List<CacheData> dataEntries = cacheEntry.getData();
|
|
for (int j = 0; j < dataEntries.size(); j++) {
|
|
CacheData data = dataEntries.get(j);
|
|
|
|
// Todo: extract the data if we are going to do something with it in the future
|
|
|
|
//data.extract();
|
|
String dataFilename = data.getAddress().getFilename();
|
|
Optional<AbstractFile> dataFile = this.findCacheFile(dataFilename);
|
|
|
|
Collection<BlackboardAttribute> bbattributes = new ArrayList<>();
|
|
bbattributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_URL,
|
|
moduleName,
|
|
((cacheEntry.getKey() != null) ? cacheEntry.getKey() : ""))); //NON-NLS
|
|
|
|
if (dataFile.isPresent()) {
|
|
if (data.isInExternalFile() ) {
|
|
try {
|
|
BlackboardArtifact bbart = dataFile.get().newArtifact(ARTIFACT_TYPE.TSK_SOURCE_ARTIFACT);
|
|
if (bbart != null) {
|
|
bbart.addAttributes(bbattributes);
|
|
}
|
|
} catch (TskException ex) {
|
|
logger.log(Level.SEVERE, "Error while trying to add an artifact", ex); //NON-NLS
|
|
}
|
|
} else {
|
|
|
|
// extract data segment and save it as derived file
|
|
data.extract();
|
|
String filename = data.save();
|
|
String relPathname = getRelOutputFolderName() + File.separator + filename;
|
|
|
|
// TBD: check if data segment is compressed? With Brotli?
|
|
DerivedFile derivedFile = fileManager.addDerivedFile(filename, relPathname,
|
|
data.getDataLength(),
|
|
cacheEntry.getCreationTime(), cacheEntry.getCreationTime(), cacheEntry.getCreationTime(), cacheEntry.getCreationTime(), // TBD
|
|
true,
|
|
dataFile.get(),
|
|
"",
|
|
moduleName,
|
|
VERSION_NUMBER,
|
|
"",
|
|
TskData.EncodingType.NONE);
|
|
|
|
derivedFiles.add(derivedFile);
|
|
try {
|
|
BlackboardArtifact bbart = derivedFile.newArtifact(ARTIFACT_TYPE.TSK_SOURCE_ARTIFACT);
|
|
if (bbart != null) {
|
|
bbart.addAttributes(bbattributes);
|
|
}
|
|
} catch (TskException ex) {
|
|
logger.log(Level.SEVERE, "Error while trying to add an artifact", ex); //NON-NLS
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
} catch (TskCoreException | IngestModuleException ex) {
|
|
logger.log(Level.SEVERE, String.format("Failed to get cache entry at address %s", addr)); //NON-NLS
|
|
}
|
|
}
|
|
}
|
|
|
|
if (derivedFiles.isEmpty() == false) {
|
|
for (AbstractFile derived : derivedFiles) {
|
|
services.fireModuleContentEvent(new ModuleContentEvent(derived));
|
|
}
|
|
}
|
|
|
|
context.addFilesToJob(derivedFiles);
|
|
services.fireModuleDataEvent(new ModuleDataEvent(moduleName, BlackboardArtifact.ARTIFACT_TYPE.TSK_SOURCE_ARTIFACT));
|
|
|
|
cleanup();
|
|
}
|
|
|
|
/**
|
|
* Finds abstract file for cache file with a specified name
|
|
*
|
|
* @param cacheFileName
|
|
* @return Opt
|
|
* @throws TskCoreException
|
|
*/
|
|
Optional<AbstractFile> findCacheFile(String cacheFileName) throws TskCoreException {
|
|
|
|
List<AbstractFile> cacheFiles = fileManager.findFiles(dataSource, cacheFileName, "default/cache"); //NON-NLS
|
|
if (!cacheFiles.isEmpty()) {
|
|
if (cacheFiles.size() > 1 ) {
|
|
logger.log(Level.WARNING, String.format("Found multiple matches for filename = %s", cacheFileName));
|
|
}
|
|
return Optional.of(cacheFiles.get(0));
|
|
}
|
|
|
|
return Optional.empty();
|
|
}
|
|
|
|
/**
|
|
* Returns CacheFileCopy for the specified file from the file table.
|
|
* Find the file and creates a copy if it isnt already in the table.
|
|
*
|
|
* @param cacheFileName
|
|
* @return CacheFileCopy
|
|
* @throws TskCoreException
|
|
*/
|
|
Optional<CacheFileCopy> getCacheFileCopy(String cacheFileName) throws TskCoreException, IngestModuleException {
|
|
|
|
// Check if the file is already in the table
|
|
if (filesTable.containsKey(cacheFileName)) {
|
|
return Optional.of(filesTable.get(cacheFileName));
|
|
}
|
|
|
|
return findAndCopyCacheFile(cacheFileName);
|
|
}
|
|
|
|
/**
|
|
* Finds the specified cache file and makes a temporary copy.
|
|
*
|
|
* @param cacheFileName
|
|
* @return Cache file copy
|
|
* @throws TskCoreException
|
|
*/
|
|
Optional<CacheFileCopy> findAndCopyCacheFile(String cacheFileName) throws TskCoreException, IngestModuleException {
|
|
|
|
Optional<AbstractFile> cacheFileOptional = findCacheFile(cacheFileName);
|
|
if (!cacheFileOptional.isPresent()) {
|
|
return Optional.empty();
|
|
}
|
|
|
|
AbstractFile cacheFile = cacheFileOptional.get();
|
|
String tempFilePathname = RAImageIngestModule.getRATempPath(currentCase, moduleName) + File.separator + cacheFile.getName(); //NON-NLS
|
|
try {
|
|
File newFile = new File(tempFilePathname);
|
|
ContentUtils.writeToFile(cacheFile, newFile, context::dataSourceIngestIsCancelled);
|
|
|
|
RandomAccessFile randomAccessFile = new RandomAccessFile(tempFilePathname, "r");
|
|
FileChannel roChannel = randomAccessFile.getChannel();
|
|
ByteBuffer cacheFileROBuf = roChannel.map(FileChannel.MapMode.READ_ONLY, 0,
|
|
(int) roChannel.size());
|
|
|
|
cacheFileROBuf.order(ByteOrder.nativeOrder());
|
|
CacheFileCopy cacheFileCopy = new CacheFileCopy(cacheFile, randomAccessFile, cacheFileROBuf );
|
|
|
|
if (!cacheFileName.startsWith("f_")) {
|
|
filesTable.put(cacheFileName, cacheFileCopy);
|
|
}
|
|
|
|
return Optional.of(cacheFileCopy);
|
|
}
|
|
catch (ReadContentInputStream.ReadContentInputStreamException ex) {
|
|
String msg = String.format("Error reading Chrome cache file '%s' (id=%d).",
|
|
cacheFile.getName(), cacheFile.getId());
|
|
throw new IngestModuleException(msg, ex);
|
|
} catch (IOException ex) {
|
|
String msg = String.format("Error writing temp Chrome cache file '%s' (id=%d).",
|
|
cacheFile.getName(), cacheFile.getId());
|
|
throw new IngestModuleException(msg, ex);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Encapsulates the header found in the index file
|
|
*/
|
|
final class IndexFileHeader {
|
|
|
|
private final long magic;
|
|
private final int version;
|
|
private final int numEntries;
|
|
private final int numBytes;
|
|
private final int lastFile;
|
|
private final int tableLen;
|
|
|
|
IndexFileHeader(ByteBuffer indexFileROBuf) {
|
|
|
|
magic = indexFileROBuf.getInt() & UINT32_MASK;
|
|
|
|
indexFileROBuf.position(indexFileROBuf.position()+2);
|
|
|
|
version = indexFileROBuf.getShort();
|
|
numEntries = indexFileROBuf.getInt();
|
|
numBytes = indexFileROBuf.getInt();
|
|
lastFile = indexFileROBuf.getInt();
|
|
|
|
indexFileROBuf.position(indexFileROBuf.position()+4); // this_id
|
|
indexFileROBuf.position(indexFileROBuf.position()+4); // stats cache address
|
|
|
|
tableLen = indexFileROBuf.getInt();
|
|
}
|
|
|
|
public long getMagic() {
|
|
return magic;
|
|
}
|
|
|
|
public int getVersion() {
|
|
return version;
|
|
}
|
|
|
|
public int getNumEntries() {
|
|
return numEntries;
|
|
}
|
|
|
|
public int getNumBytes() {
|
|
return numBytes;
|
|
}
|
|
|
|
public int getLastFile() {
|
|
return lastFile;
|
|
}
|
|
|
|
public int getTableLen() {
|
|
return tableLen;
|
|
}
|
|
|
|
@Override
|
|
public String toString() {
|
|
StringBuilder sb = new StringBuilder();
|
|
|
|
sb.append(String.format("Index Header:"));
|
|
sb.append(String.format("\tMagic = %x" , getMagic()) );
|
|
sb.append(String.format("\tVersion = %x" , getVersion()) );
|
|
sb.append(String.format("\tNumEntries = %x" , getNumEntries()) );
|
|
sb.append(String.format("\tNumBytes = %x" , getNumBytes()) );
|
|
sb.append(String.format("\tLastFile = %x" , getLastFile()) );
|
|
sb.append(String.format("\tTableLen = %x" , getTableLen()) );
|
|
|
|
return sb.toString();
|
|
}
|
|
}
|
|
|
|
enum CacheFileTypeEnum {
|
|
EXTERNAL,
|
|
RANKINGS,
|
|
BLOCK_256,
|
|
BLOCK_1K,
|
|
BLOCK_4K,
|
|
BLOCK_FILES,
|
|
BLOCK_ENTRIES,
|
|
BLOCK_EVICTED
|
|
}
|
|
|
|
// CacheAddress is a unsigned 32 bit number
|
|
//
|
|
// Header:
|
|
// 1000 0000 0000 0000 0000 0000 0000 0000 : initialized bit
|
|
// 0111 0000 0000 0000 0000 0000 0000 0000 : file type
|
|
//
|
|
// If separate file:
|
|
// 0000 1111 1111 1111 1111 1111 1111 1111 : file# 0 - 268,435,456 (2^28)
|
|
//
|
|
// If block file:
|
|
// 0000 1100 0000 0000 0000 0000 0000 0000 : reserved bits
|
|
// 0000 0011 0000 0000 0000 0000 0000 0000 : number of contiguous blocks 1-4
|
|
// 0000 0000 1111 1111 0000 0000 0000 0000 : file selector 0 - 255
|
|
// 0000 0000 0000 0000 1111 1111 1111 1111 : block# 0 - 65,535 (2^16)
|
|
|
|
final class CacheAddress {
|
|
// sundry constants to parse the bit fields in address
|
|
private final long ADDR_INITIALIZED_MASK = 0x80000000l;
|
|
private final long FILE_TYPE_MASK = 0x70000000;
|
|
private final long FILE_TYPE_OFFSET = 28;
|
|
private final long NUM_BLOCKS_MASK = 0x03000000;
|
|
private final long NUM_BLOCKS_OFFSET = 24;
|
|
private final long FILE_SELECTOR_MASK = 0x00ff0000;
|
|
private final long FILE_SELECTOR_OFFSET = 16;
|
|
private final long START_BLOCK_MASK = 0x0000FFFF;
|
|
private final long EXTERNAL_FILE_NAME_MASK = 0x0FFFFFFF;
|
|
|
|
private final long uint32CacheAddr;
|
|
private final CacheFileTypeEnum fileType;
|
|
private final int numBlocks;
|
|
private final int startBlock;
|
|
private final String fileName;
|
|
private final int fileNumber;
|
|
|
|
|
|
CacheAddress(long uint32) {
|
|
|
|
uint32CacheAddr = uint32;
|
|
int fileTypeEnc = (int)(uint32CacheAddr & FILE_TYPE_MASK) >> FILE_TYPE_OFFSET;
|
|
fileType = CacheFileTypeEnum.values()[fileTypeEnc];
|
|
|
|
if (isInitialized()) {
|
|
if (isInExternalFile()) {
|
|
fileNumber = (int)(uint32CacheAddr & EXTERNAL_FILE_NAME_MASK);
|
|
fileName = String.format("f_%06x", getFileNumber() );
|
|
numBlocks = 0;
|
|
startBlock = 0;
|
|
} else {
|
|
fileNumber = (int)((uint32CacheAddr & FILE_SELECTOR_MASK) >> FILE_SELECTOR_OFFSET);
|
|
fileName = String.format("data_%d", getFileNumber() );
|
|
numBlocks = (int)(uint32CacheAddr & NUM_BLOCKS_MASK >> NUM_BLOCKS_OFFSET);
|
|
startBlock = (int)(uint32CacheAddr & START_BLOCK_MASK);
|
|
}
|
|
}
|
|
else {
|
|
fileName = null;
|
|
fileNumber = 0;
|
|
numBlocks = 0;
|
|
startBlock = 0;
|
|
}
|
|
}
|
|
|
|
boolean isInitialized() {
|
|
return ((uint32CacheAddr & ADDR_INITIALIZED_MASK) != 0);
|
|
}
|
|
|
|
CacheFileTypeEnum getFileType() {
|
|
return fileType;
|
|
}
|
|
|
|
String getFilename() {
|
|
return fileName;
|
|
}
|
|
|
|
boolean isInExternalFile() {
|
|
return (fileType == CacheFileTypeEnum.EXTERNAL);
|
|
}
|
|
|
|
int getFileNumber() {
|
|
return fileNumber;
|
|
}
|
|
|
|
int getStartBlock() {
|
|
return startBlock;
|
|
}
|
|
|
|
int getNumBlocks() {
|
|
return numBlocks;
|
|
}
|
|
|
|
int getBlockSize() {
|
|
switch (fileType) {
|
|
case RANKINGS:
|
|
return 36;
|
|
case BLOCK_256:
|
|
return 256;
|
|
case BLOCK_1K:
|
|
return 1024;
|
|
case BLOCK_4K:
|
|
return 4096;
|
|
case BLOCK_FILES:
|
|
return 8;
|
|
case BLOCK_ENTRIES:
|
|
return 104;
|
|
case BLOCK_EVICTED:
|
|
return 48;
|
|
default:
|
|
return 0;
|
|
}
|
|
}
|
|
|
|
public long getUint32CacheAddr() {
|
|
return uint32CacheAddr;
|
|
}
|
|
|
|
@Override
|
|
public String toString() {
|
|
StringBuilder sb = new StringBuilder();
|
|
sb.append(String.format("CacheAddr %08x : %s : filename %s",
|
|
uint32CacheAddr,
|
|
isInitialized() ? "Initialized" : "UnInitialized",
|
|
getFilename()));
|
|
|
|
if ((fileType == CacheFileTypeEnum.BLOCK_256) ||
|
|
(fileType == CacheFileTypeEnum.BLOCK_1K) ||
|
|
(fileType == CacheFileTypeEnum.BLOCK_4K) ) {
|
|
sb.append(String.format(" (%d blocks starting at %08X)",
|
|
this.getNumBlocks(),
|
|
this.getStartBlock()
|
|
));
|
|
}
|
|
|
|
return sb.toString();
|
|
}
|
|
|
|
}
|
|
|
|
|
|
enum CacheDataTypeEnum {
|
|
HTTP_HEADER,
|
|
UNKNOWN,
|
|
};
|
|
|
|
/**
|
|
* Encapsulates a cached data segment.
|
|
*
|
|
* A data segment may have HTTP headers, scripts, image files (png, gif), or JSON files
|
|
*
|
|
* A data segment may be stored in one of the data_x files or an external file - f_xxxxxx
|
|
*
|
|
* A data segment may be compressed - GZIP and BRotli are the two commonly used methods.
|
|
*/
|
|
final class CacheData {
|
|
|
|
private int length;
|
|
private final CacheAddress address;
|
|
private CacheDataTypeEnum type;
|
|
|
|
private boolean isHTTPHeaderHint;
|
|
|
|
private CacheFileCopy cacheFileCopy = null;
|
|
private byte[] data = null;
|
|
|
|
// mime type of the data segment helps determine if it is compressed
|
|
private String mimeType = "";
|
|
|
|
CacheData(CacheAddress cacheAdress, int len) {
|
|
this(cacheAdress, len, false);
|
|
}
|
|
|
|
CacheData(CacheAddress cacheAdress, int len, boolean isHTTPHeader ) {
|
|
this.type = CacheDataTypeEnum.UNKNOWN;
|
|
this.length = len;
|
|
this.address = cacheAdress;
|
|
this.isHTTPHeaderHint = isHTTPHeader;
|
|
}
|
|
|
|
boolean isInExternalFile() {
|
|
return address.isInExternalFile();
|
|
}
|
|
|
|
boolean isCompressedFile() {
|
|
if (isInExternalFile()) {
|
|
return mimeType.equalsIgnoreCase("application/octet-stream");
|
|
}
|
|
else {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
String getMimeType() {
|
|
return mimeType;
|
|
}
|
|
|
|
/**
|
|
* Extracts the data segment from the file
|
|
*
|
|
* @throws TskCoreException
|
|
*/
|
|
void extract() throws TskCoreException, IngestModuleException {
|
|
|
|
// do nothing if already extracted,
|
|
if (data != null) {
|
|
return;
|
|
}
|
|
|
|
cacheFileCopy = getCacheFileCopy(address.getFilename()).get();
|
|
if (!address.isInExternalFile() ) {
|
|
|
|
this.data = new byte [length];
|
|
ByteBuffer buf = cacheFileCopy.getByteBuffer();
|
|
int dataOffset = DATAFILE_HDR_SIZE + address.getStartBlock() * address.getBlockSize();
|
|
buf.position(dataOffset);
|
|
buf.get(data, 0, length);
|
|
|
|
// if this might be a HTPP header, lets try to parse it as such
|
|
if ((isHTTPHeaderHint)) {
|
|
// Check if we can find the http headers
|
|
String strData = new String(data);
|
|
if (strData.contains("HTTP")) {
|
|
|
|
// TBD parse header
|
|
// Past some bytes there's the HTTP headers
|
|
// General Parsing algo:
|
|
// - Find start of HTTP header by searching for string "HTTP"
|
|
// - Skip to the first 0x00 ti get to the end of the HTTP response line, this makrs start of headers section
|
|
// - Find the end of the end by searching for 0x00 0x00 bytes
|
|
// - Extract the headers section
|
|
// - Parse the headers section - each null terminated string is a header
|
|
// - Each header is of the format "name: value" e.g.
|
|
|
|
type = CacheDataTypeEnum.HTTP_HEADER;
|
|
}
|
|
}
|
|
|
|
} else {
|
|
// Handle external f_* files
|
|
|
|
// External files may or may not be compressed
|
|
// They may be compresswed with GZIP, which our other ingest modules recognize and decpress
|
|
// Alternatively thay may be compressed with Brotli, in that case we may want to decopmress them
|
|
|
|
// TBD: In future if we want to do anything with contents of file.
|
|
// this.data = new byte [length];
|
|
//
|
|
// ByteBuffer buf = cacheFileCopy.getByteBuffer();
|
|
// buf.position(0);
|
|
// buf.get(data, 0, length);
|
|
//
|
|
// // get mime type, to determine if the file is compressed or not
|
|
// AbstractFile abstractFile = cacheFileCopy.getAbstractFile();
|
|
// mimeType = fileTypeDetector.getMIMEType(abstractFile);
|
|
|
|
}
|
|
}
|
|
|
|
String getDataString() throws TskCoreException, IngestModuleException {
|
|
if (data == null) {
|
|
extract();
|
|
}
|
|
return new String(data);
|
|
}
|
|
|
|
byte[] getDataBytes() throws TskCoreException, IngestModuleException {
|
|
if (data == null) {
|
|
extract();
|
|
}
|
|
return data;
|
|
}
|
|
|
|
int getDataLength() {
|
|
return this.length;
|
|
}
|
|
|
|
CacheDataTypeEnum getType() {
|
|
return type;
|
|
}
|
|
|
|
CacheAddress getAddress() {
|
|
return address;
|
|
}
|
|
|
|
|
|
String save() throws TskCoreException, IngestModuleException {
|
|
String fileName;
|
|
|
|
if (address.isInExternalFile()) {
|
|
fileName = address.getFilename();
|
|
} else {
|
|
fileName = String.format("%s__%08x", address.getFilename(), address.getUint32CacheAddr());
|
|
}
|
|
|
|
String filePathName = getAbsOutputFolderName() + File.separator + fileName;
|
|
save(filePathName);
|
|
|
|
return fileName;
|
|
}
|
|
|
|
|
|
void save(String filePathName) throws TskCoreException, IngestModuleException {
|
|
|
|
// Save the data to specified file
|
|
if (data == null) {
|
|
extract();
|
|
}
|
|
|
|
if (!this.isInExternalFile() ||
|
|
!this.isCompressedFile()) {
|
|
|
|
// write the
|
|
try (FileOutputStream stream = new FileOutputStream(filePathName)) {
|
|
stream.write(data);
|
|
} catch (IOException ex) {
|
|
throw new TskCoreException(String.format("Failed to write output file %s", filePathName), ex);
|
|
}
|
|
}
|
|
// else {
|
|
// if (mimeType.toLowerCase().contains("gzip")) {
|
|
// //if (mimeType.equalsIgnoreCase("application/gzip")) {
|
|
// try {
|
|
// ByteArrayInputStream byteInputStream = new ByteArrayInputStream(data);
|
|
// GZIPInputStream in = new GZIPInputStream(byteInputStream);
|
|
// FileOutputStream out = new FileOutputStream(filePathName);
|
|
// byte[] buffer = new byte[2048];
|
|
// int len;
|
|
// while((len = in.read(buffer)) != -1){
|
|
// out.write(buffer, 0, len);
|
|
// }
|
|
// out.close();
|
|
//
|
|
// } catch (IOException ex) {
|
|
// throw new TskCoreException(String.format("Failed to write output file %s", filePathName), ex);
|
|
// }
|
|
// }
|
|
// else {
|
|
// // TBD: how to uncompress Brotli ??
|
|
// System.out.println("TBD Dont know how to uncompress Brotli yet" );
|
|
// }
|
|
// }
|
|
}
|
|
|
|
@Override
|
|
public String toString() {
|
|
StringBuilder sb = new StringBuilder();
|
|
sb.append(String.format("\t\tData type = : %s, Data Len = %d",
|
|
this.type.toString(), this.length ));
|
|
sb.append("\n");
|
|
sb.append(String.format("\t\tData = : %s ", new String(data) ));
|
|
|
|
return sb.toString();
|
|
}
|
|
|
|
}
|
|
|
|
|
|
enum EntryStateEnum {
|
|
ENTRY_NORMAL,
|
|
ENTRY_EVICTED,
|
|
ENTRY_DOOMED
|
|
};
|
|
|
|
|
|
// Main structure for an entry on the backing storage.
|
|
//
|
|
// Each entry has a key, identifying the URL the cache entry pertains to.
|
|
// If the key is longer than
|
|
// what can be stored on this structure, it will be extended on consecutive
|
|
// blocks (adding 256 bytes each time), up to 4 blocks (1024 - 32 - 1 chars).
|
|
// After that point, the whole key will be stored as a data block or external
|
|
// file.
|
|
//
|
|
// Each entry can have upto 4 data segments
|
|
//
|
|
// struct EntryStore {
|
|
// uint32 hash; // Full hash of the key.
|
|
// CacheAddr next; // Next entry with the same hash or bucket.
|
|
// CacheAddr rankings_node; // Rankings node for this entry.
|
|
// int32 reuse_count; // How often is this entry used.
|
|
// int32 refetch_count; // How often is this fetched from the net.
|
|
// int32 state; // Current state.
|
|
// uint64 creation_time;
|
|
// int32 key_len;
|
|
// CacheAddr long_key; // Optional address of a long key.
|
|
// int32 data_size[4]; // We can store up to 4 data streams for each
|
|
// CacheAddr data_addr[4]; // entry.
|
|
// uint32 flags; // Any combination of EntryFlags.
|
|
// int32 pad[4];
|
|
// uint32 self_hash; // The hash of EntryStore up to this point.
|
|
// char key[256 - 24 * 4]; // null terminated
|
|
// };
|
|
|
|
/**
|
|
* Encapsulates a Cache Entry
|
|
*/
|
|
final class CacheEntry {
|
|
|
|
// each entry is 256 bytes. The last section of the entry, after all the other fields is a null terminated key
|
|
private final int MAX_KEY_LEN = 256-24*4;
|
|
|
|
private final CacheAddress selfAddress;
|
|
private final CacheFileCopy cacheFileCopy;
|
|
|
|
private long hash;
|
|
private CacheAddress nextAddress;
|
|
private CacheAddress rankingsNodeAddress;
|
|
|
|
private int reuseCount;
|
|
private int refetchCount;
|
|
private EntryStateEnum state;
|
|
|
|
private long creationTime;
|
|
private int keyLen;
|
|
|
|
private CacheAddress longKeyAddresses; // address of the key, if the key is external to the entry
|
|
|
|
private int dataSizes[] = new int[4];
|
|
private CacheAddress dataAddresses[] = new CacheAddress[4];
|
|
|
|
private long flags;
|
|
private int pad[] = new int[4];
|
|
|
|
private long selfHash; // hash of the entry itself so far.
|
|
private String key; // Key may be found within the entry or may be external
|
|
|
|
CacheEntry(CacheAddress cacheAdress, CacheFileCopy cacheFileCopy ) {
|
|
this.selfAddress = cacheAdress;
|
|
this.cacheFileCopy = cacheFileCopy;
|
|
|
|
ByteBuffer fileROBuf = cacheFileCopy.getByteBuffer();
|
|
|
|
int entryOffset = DATAFILE_HDR_SIZE + cacheAdress.getStartBlock() * cacheAdress.getBlockSize();
|
|
|
|
// reposition the buffer to the the correct offset
|
|
fileROBuf.position(entryOffset);
|
|
|
|
hash = fileROBuf.getInt() & UINT32_MASK;
|
|
|
|
long uint32 = fileROBuf.getInt() & UINT32_MASK;
|
|
nextAddress = (uint32 != 0) ? new CacheAddress(uint32) : null;
|
|
|
|
uint32 = fileROBuf.getInt() & UINT32_MASK;
|
|
rankingsNodeAddress = (uint32 != 0) ? new CacheAddress(uint32) : null;
|
|
|
|
reuseCount = fileROBuf.getInt();
|
|
refetchCount = fileROBuf.getInt();
|
|
|
|
state = EntryStateEnum.values()[fileROBuf.getInt()];
|
|
creationTime = (fileROBuf.getLong() / 1000000) - Long.valueOf("11644473600");
|
|
|
|
keyLen = fileROBuf.getInt();
|
|
|
|
uint32 = fileROBuf.getInt() & UINT32_MASK;
|
|
longKeyAddresses = (uint32 != 0) ? new CacheAddress(uint32) : null;
|
|
|
|
for (int i = 0; i < 4; i++) {
|
|
dataSizes[i] = fileROBuf.getInt();
|
|
}
|
|
for (int i = 0; i < 4; i++) {
|
|
dataAddresses[i] = new CacheAddress(fileROBuf.getInt() & UINT32_MASK);
|
|
}
|
|
|
|
flags = fileROBuf.getInt() & UINT32_MASK;
|
|
for (int i = 0; i < 4; i++) {
|
|
pad[i] = fileROBuf.getInt();
|
|
}
|
|
|
|
selfHash = fileROBuf.getInt() & UINT32_MASK;
|
|
|
|
// get the key
|
|
if (longKeyAddresses != null) {
|
|
// Key is stored outside of the entry
|
|
try {
|
|
CacheData data = new CacheData(longKeyAddresses, this.keyLen, true);
|
|
key = data.getDataString();
|
|
} catch (TskCoreException | IngestModuleException ex) {
|
|
logger.log(Level.SEVERE, String.format("Failed to get external key from address %s", longKeyAddresses)); //NON-NLS
|
|
}
|
|
}
|
|
else { // key stored within entry
|
|
StringBuilder sb = new StringBuilder(MAX_KEY_LEN);
|
|
int i = 0;
|
|
while (fileROBuf.remaining() > 0 && i < MAX_KEY_LEN) {
|
|
char c = (char)fileROBuf.get();
|
|
if (c == '\0') {
|
|
break;
|
|
}
|
|
sb.append(c);
|
|
}
|
|
|
|
key = sb.toString();
|
|
}
|
|
}
|
|
|
|
public CacheAddress getAddress() {
|
|
return selfAddress;
|
|
}
|
|
|
|
public long getHash() {
|
|
return hash;
|
|
}
|
|
|
|
public CacheAddress getNextAddress() {
|
|
return nextAddress;
|
|
}
|
|
|
|
public int getReuseCount() {
|
|
return reuseCount;
|
|
}
|
|
|
|
public int getRefetchCount() {
|
|
return refetchCount;
|
|
}
|
|
|
|
public EntryStateEnum getState() {
|
|
return state;
|
|
}
|
|
|
|
public long getCreationTime() {
|
|
return creationTime;
|
|
}
|
|
|
|
public long getFlags() {
|
|
return flags;
|
|
}
|
|
|
|
public String getKey() {
|
|
return key;
|
|
}
|
|
|
|
public ArrayList<CacheData> getData() {
|
|
ArrayList<CacheData> list = new ArrayList<>();
|
|
for (int i = 0; i < 4; i++) {
|
|
if (dataSizes[i] > 0) {
|
|
CacheData cacheData = new CacheData(dataAddresses[i], dataSizes[i], true );
|
|
list.add(cacheData);
|
|
}
|
|
}
|
|
return list;
|
|
}
|
|
|
|
@Override
|
|
public String toString() {
|
|
StringBuilder sb = new StringBuilder();
|
|
sb.append(String.format("Entry = Hash: %08x, State: %s, ReuseCount: %d, RefetchCount: %d",
|
|
this.hash, this.state.toString(), this.reuseCount, this.refetchCount ));
|
|
|
|
sb.append(String.format("\n\tKey: %s, Keylen: %d",
|
|
this.key, this.keyLen, this.reuseCount, this.refetchCount ));
|
|
|
|
sb.append(String.format("\n\tCreationTime: %s",
|
|
TimeUtilities.epochToTime(this.creationTime) ));
|
|
|
|
sb.append(String.format("\n\tNext Address: %s",
|
|
(nextAddress != null) ? nextAddress.toString() : "None"));
|
|
|
|
for (int i = 0; i < 4; i++) {
|
|
if (dataSizes[i] > 0) {
|
|
sb.append(String.format("\n\tData %d: %8d bytes at cache address = %s",
|
|
i, dataSizes[i], dataAddresses[i] ));
|
|
}
|
|
}
|
|
|
|
return sb.toString();
|
|
}
|
|
}
|
|
}
|