/* * * Autopsy Forensic Browser * * Copyright 2012-2013 Basis Technology Corp. * * Copyright 2012 42six Solutions. * Contact: aebadirad 42six com * Project Contact/Architect: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.sleuthkit.autopsy.recentactivity; //IO imports import org.sleuthkit.autopsy.coreutils.ExecUtil; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileWriter; import java.io.IOException; import java.io.Writer; //Util Imports import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.List; import java.util.logging.Level; import org.sleuthkit.autopsy.coreutils.Logger; import java.util.Collection; import java.util.Scanner; import java.util.regex.Matcher; import java.util.regex.Pattern; // TSK Imports import org.openide.modules.InstalledFileLocator; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.coreutils.EscapeUtil; import org.sleuthkit.autopsy.coreutils.JLNK; import org.sleuthkit.autopsy.coreutils.JLnkParser; import org.sleuthkit.autopsy.datamodel.ContentUtils; import org.sleuthkit.autopsy.ingest.IngestDataSourceWorkerController; import org.sleuthkit.autopsy.ingest.IngestServices; import org.sleuthkit.autopsy.ingest.ModuleDataEvent; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE; import org.sleuthkit.datamodel.BlackboardAttribute; import org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE; import org.sleuthkit.datamodel.Content; import org.sleuthkit.autopsy.coreutils.PlatformUtil; import org.sleuthkit.autopsy.ingest.PipelineContext; import org.sleuthkit.autopsy.ingest.IngestModuleDataSource; import org.sleuthkit.autopsy.ingest.IngestModuleInit; import org.sleuthkit.datamodel.*; public class ExtractIE extends Extract { private static final Logger logger = Logger.getLogger(ExtractIE.class.getName()); private IngestServices services; //paths set in init() private String moduleTempResultsDir; private String PASCO_LIB_PATH; private String JAVA_PATH; // List of Pasco result files for this data source private List pascoResults; boolean pascoFound = false; final public static String MODULE_VERSION = "1.0"; private static final SimpleDateFormat dateFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); private ExecUtil execPasco; //hide public constructor to prevent from instantiation by ingest module loader ExtractIE() { moduleName = "Internet Explorer"; moduleTempResultsDir = RAImageIngestModule.getRATempPath(Case.getCurrentCase(), "IE") + File.separator + "results"; JAVA_PATH = PlatformUtil.getJavaPath(); } @Override public String getVersion() { return MODULE_VERSION; } @Override public void process(PipelineContextpipelineContext, Content dataSource, IngestDataSourceWorkerController controller) { /* @@@ BC: All of these try / catches are added because the exception * handing in here isn't the best. We were losing results before because * cookies was throwing an exceptionb ecause of an invalid URL and we * skipped the rest of the data types. Need to push this eror handling * further down though. */ try { this.extractAndRunPasco(dataSource, controller); } catch (Exception e) { logger.log(Level.SEVERE, "Error extracting IE index.dat files", e); addErrorMessage("Error extracting and analyzing IE index.dat files"); } try { this.getBookmark(dataSource, controller); } catch (Exception e) { logger.log(Level.SEVERE, "Error parsing IE bookmarks", e); addErrorMessage("Error parsing IE bookmarks"); } try { this.getCookie(dataSource, controller); } catch (Exception e) { logger.log(Level.SEVERE, "Error parsing IE cookies", e); addErrorMessage("Error parsing IE Cookies"); } try { this.getRecentDocuments(dataSource, controller); } catch (Exception e) { logger.log(Level.SEVERE, "Error parsing IE Recent Docs", e); addErrorMessage("Error parsing IE Recent Documents"); } try { this.getHistory(pascoResults); } catch (Exception e) { logger.log(Level.SEVERE, "Error parsing IE History", e); addErrorMessage("Error parsing IE History"); } } //Favorites section // This gets the favorite info private void getBookmark(Content dataSource, IngestDataSourceWorkerController controller) { int errors = 0; org.sleuthkit.autopsy.casemodule.services.FileManager fileManager = currentCase.getServices().getFileManager(); List favoritesFiles = null; try { favoritesFiles = fileManager.findFiles(dataSource, "%.url", "Favorites"); } catch (TskCoreException ex) { logger.log(Level.WARNING, "Error fetching 'index.data' files for Internet Explorer history."); } for (AbstractFile favoritesFile : favoritesFiles) { if (controller.isCancelled()) { break; } Content fav = favoritesFile; byte[] t = new byte[(int) fav.getSize()]; try { final int bytesRead = fav.read(t, 0, fav.getSize()); } catch (TskCoreException ex) { logger.log(Level.SEVERE, "Error reading bytes of Internet Explorer favorite.", ex); } String bookmarkString = new String(t); String re1 = ".*?"; // Non-greedy match on filler String re2 = "((?:http|https)(?::\\/{2}[\\w]+)(?:[\\/|\\.]?)(?:[^\\s\"]*))"; // HTTP URL 1 String url = ""; Pattern p = Pattern.compile(re1 + re2, Pattern.CASE_INSENSITIVE | Pattern.DOTALL); Matcher m = p.matcher(bookmarkString); if (m.find()) { url = m.group(1); } String name = favoritesFile.getName(); Long datetime = favoritesFile.getCrtime(); String Tempdate = datetime.toString(); datetime = Long.valueOf(Tempdate); String domain = Util.extractDomain(url); Collection bbattributes = new ArrayList(); //TODO revisit usage of deprecated constructor as per TSK-583 //bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_LAST_ACCESSED.getTypeID(), "RecentActivity", "Last Visited", datetime)); bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_ACCESSED.getTypeID(), "RecentActivity", datetime)); bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL.getTypeID(), "RecentActivity", url)); //bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL_DECODED.getTypeID(), "RecentActivity", EscapeUtil.decodeURL(url))); bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_NAME.getTypeID(), "RecentActivity", name)); bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PROG_NAME.getTypeID(), "RecentActivity", "Internet Explorer")); bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DOMAIN.getTypeID(), "RecentActivity", domain)); this.addArtifact(ARTIFACT_TYPE.TSK_WEB_BOOKMARK, favoritesFile, bbattributes); services.fireModuleDataEvent(new ModuleDataEvent("Recent Activity", BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_BOOKMARK)); } if (errors > 0) { this.addErrorMessage(this.getName() + ": Error parsing " + errors + " Internet Explorer favorites."); } } //Cookies section // This gets the cookies info private void getCookie(Content dataSource, IngestDataSourceWorkerController controller) { org.sleuthkit.autopsy.casemodule.services.FileManager fileManager = currentCase.getServices().getFileManager(); List cookiesFiles = null; try { cookiesFiles = fileManager.findFiles(dataSource, "%.txt", "Cookies"); } catch (TskCoreException ex) { logger.log(Level.WARNING, "Error fetching 'index.data' files for Internet Explorer history."); } int errors = 0; for (AbstractFile cookiesFile : cookiesFiles) { if (controller.isCancelled()) { break; } Content fav = cookiesFile; byte[] t = new byte[(int) fav.getSize()]; try { final int bytesRead = fav.read(t, 0, fav.getSize()); } catch (TskCoreException ex) { logger.log(Level.SEVERE, "Error reading bytes of Internet Explorer cookie.", ex); } String cookieString = new String(t); String[] values = cookieString.split("\n"); String url = values.length > 2 ? values[2] : ""; String value = values.length > 1 ? values[1] : ""; String name = values.length > 0 ? values[0] : ""; Long datetime = cookiesFile.getCrtime(); String Tempdate = datetime.toString(); datetime = Long.valueOf(Tempdate); String domain = Util.extractDomain(url); Collection bbattributes = new ArrayList(); bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL.getTypeID(), "RecentActivity", url)); //bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL_DECODED.getTypeID(), "RecentActivity", EscapeUtil.decodeURL(url))); //TODO Revisit usage of deprecated Constructor as of TSK-583 //bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME.getTypeID(), "RecentActivity", "Last Visited", datetime)); bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME.getTypeID(), "RecentActivity", datetime)); bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_VALUE.getTypeID(), "RecentActivity", value)); //TODO Revisit usage of deprecated Constructor as of TSK-583 //bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_NAME.getTypeID(), "RecentActivity", "Title", (name != null) ? name : "")); bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_NAME.getTypeID(), "RecentActivity", (name != null) ? name : "")); bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PROG_NAME.getTypeID(), "RecentActivity", "Internet Explorer")); bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DOMAIN.getTypeID(), "RecentActivity", domain)); this.addArtifact(ARTIFACT_TYPE.TSK_WEB_COOKIE, cookiesFile, bbattributes); } if (errors > 0) { this.addErrorMessage(this.getName() + ": Error parsing " + errors + " Internet Explorer cookies."); } services.fireModuleDataEvent(new ModuleDataEvent("Recent Activity", BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_COOKIE)); } //Recent Documents section // This gets the recent object info private void getRecentDocuments(Content dataSource, IngestDataSourceWorkerController controller) { org.sleuthkit.autopsy.casemodule.services.FileManager fileManager = currentCase.getServices().getFileManager(); List recentFiles = null; try { recentFiles = fileManager.findFiles(dataSource, "%.lnk", "Recent"); } catch (TskCoreException ex) { logger.log(Level.WARNING, "Error fetching 'index.data' files for Internet Explorer history."); } for (AbstractFile recentFile : recentFiles) { if (controller.isCancelled()) { break; } Content fav = recentFile; if (fav.getSize() == 0) { continue; } JLNK lnk = null; JLnkParser lnkParser = new JLnkParser(new ReadContentInputStream(fav), (int) fav.getSize()); try { lnk = lnkParser.parse(); } catch (Exception e) { //TODO should throw a specific checked exception logger.log(Level.SEVERE, "Error lnk parsing the file to get recent files" + recentFile, e); this.addErrorMessage(this.getName() + ": Error parsing Recent File " + recentFile.getName()); continue; } String path = lnk.getBestPath(); Long datetime = recentFile.getCrtime(); Collection bbattributes = new ArrayList(); bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PATH.getTypeID(), "RecentActivity", path)); bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_NAME.getTypeID(), "RecentActivity", Util.getFileName(path))); long id = Util.findID(dataSource, path); bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PATH_ID.getTypeID(), "RecentActivity", id)); //TODO Revisit usage of deprecated constructor as per TSK-583 //bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME.getTypeID(), "RecentActivity", "Date Created", datetime)); bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME.getTypeID(), "RecentActivity", datetime)); this.addArtifact(ARTIFACT_TYPE.TSK_RECENT_OBJECT, recentFile, bbattributes); } services.fireModuleDataEvent(new ModuleDataEvent("Recent Activity", BlackboardArtifact.ARTIFACT_TYPE.TSK_RECENT_OBJECT)); } private void extractAndRunPasco(Content dataSource, IngestDataSourceWorkerController controller) { pascoResults = new ArrayList(); logger.log(Level.INFO, "Pasco results path: " + moduleTempResultsDir); final File pascoRoot = InstalledFileLocator.getDefault().locate("pasco2", ExtractIE.class.getPackage().getName(), false); if (pascoRoot == null) { logger.log(Level.SEVERE, "Pasco2 not found"); pascoFound = false; return; } else { pascoFound = true; } final String pascoHome = pascoRoot.getAbsolutePath(); logger.log(Level.INFO, "Pasco2 home: " + pascoHome); PASCO_LIB_PATH = pascoHome + File.separator + "pasco2.jar" + File.pathSeparator + pascoHome + File.separator + "*"; File resultsDir = new File(moduleTempResultsDir); resultsDir.mkdirs(); // get index.dat files org.sleuthkit.autopsy.casemodule.services.FileManager fileManager = currentCase.getServices().getFileManager(); List indexFiles = null; try { indexFiles = fileManager.findFiles(dataSource, "index.dat"); } catch (TskCoreException ex) { logger.log(Level.WARNING, "Error fetching 'index.data' files for Internet Explorer history."); } String temps; String indexFileName; for (AbstractFile indexFile : indexFiles) { // Since each result represent an index.dat file, // just create these files with the following notation: // index.dat (i.e. index0.dat, index1.dat,..., indexN.dat) // Write each index.dat file to a temp directory. //BlackboardArtifact bbart = fsc.newArtifact(ARTIFACT_TYPE.TSK_WEB_HISTORY); indexFileName = "index" + Integer.toString((int) indexFile.getId()) + ".dat"; //indexFileName = "index" + Long.toString(bbart.getArtifactID()) + ".dat"; temps = RAImageIngestModule.getRATempPath(currentCase, "IE") + File.separator + indexFileName; File datFile = new File(temps); if (controller.isCancelled()) { datFile.delete(); break; } try { ContentUtils.writeToFile(indexFile, datFile); } catch (IOException e) { logger.log(Level.SEVERE, "Error while trying to write index.dat file " + datFile.getAbsolutePath(), e); this.addErrorMessage(this.getName() + ": Error while trying to write file:" + datFile.getAbsolutePath()); continue; } String filename = "pasco2Result." + indexFile.getId() + ".txt"; boolean bPascProcSuccess = executePasco(temps, filename); //At this point pasco2 proccessed the index files. //Now fetch the results, parse them and the delete the files. if (bPascProcSuccess) { pascoResults.add(filename); //Delete index.dat file since it was succcessfully by Pasco datFile.delete(); } else { this.addErrorMessage(this.getName() + ": Error processing Internet Explorer history."); } } } //Simple wrapper to JavaSystemCaller.Exec() to execute pasco2 jar // TODO: Hardcoded command args/path needs to be removed. Maybe set some constants and set env variables for classpath // I'm not happy with this code. Can't stand making a system call, is not an acceptable solution but is a hack for now. private boolean executePasco(String indexFilePath, String filename) { if (pascoFound == false) { return false; } boolean success = true; Writer writer = null; try { final String pascoOutFile = moduleTempResultsDir + File.separator + filename; logger.log(Level.INFO, "Writing pasco results to: " + pascoOutFile); writer = new FileWriter(pascoOutFile); execPasco = new ExecUtil(); execPasco.execute(writer, JAVA_PATH, "-cp", PASCO_LIB_PATH, "isi.pasco2.Main", "-T", "history", indexFilePath ); // @@@ Investigate use of history versus cache as type. } catch (IOException ex) { success = false; logger.log(Level.SEVERE, "Unable to execute Pasco to process Internet Explorer web history.", ex); } catch (InterruptedException ex) { success = false; logger.log(Level.SEVERE, "Pasco has been interrupted, failed to extract some web history from Internet Explorer.", ex); } finally { if (writer != null) { try { writer.flush(); writer.close(); } catch (IOException ex) { logger.log(Level.WARNING, "Error closing writer stream after for Pasco result", ex); } } } return success; } private void getHistory(List filenames) { if (pascoFound == false) { return; } // First thing we want to do is check to make sure the results directory // is not empty. File rFile = new File(moduleTempResultsDir); if (rFile.exists()) { //Give me a list of pasco results in that directory File[] pascoFiles = rFile.listFiles(); if (pascoFiles.length > 0) { for (File file : pascoFiles) { String fileName = file.getName(); if (!filenames.contains(fileName)) { logger.log(Level.INFO, "Found a temp Pasco result file not in the list: {0}", fileName); continue; } long artObjId = Long.parseLong(fileName.substring(fileName.indexOf(".") + 1, fileName.lastIndexOf("."))); // Make sure the file the is not empty or the Scanner will // throw a "No Line found" Exception if (file != null && file.length() > 0) { try { Scanner fileScanner = new Scanner(new FileInputStream(file.toString())); //Skip the first three lines fileScanner.nextLine(); fileScanner.nextLine(); fileScanner.nextLine(); while (fileScanner.hasNext()) { String line = fileScanner.nextLine(); // lines at end of file if ((line.startsWith("LEAK entries")) || (line.startsWith("REDR entries")) || (line.startsWith("URL entries")) || (line.startsWith("ent entries")) || (line.startsWith("unknown entries"))) { continue; } if (line.startsWith("URL")) { String[] lineBuff = line.split("\\t"); if (lineBuff.length < 4) { // @@@ Could log a message here continue; } String ddtime = lineBuff[2]; String actime = lineBuff[3]; Long ftime = (long) 0; String user = ""; String realurl = ""; String domain = ""; /* We've seen two types of lines: * URL http://XYZ.com .... * URL Visited: Joe@http://XYZ.com .... */ if (lineBuff[1].contains("@")) { String url[] = lineBuff[1].split("@", 2); user = url[0]; user = user.replace("Visited:", ""); user = user.replace(":Host:", ""); user = user.replaceAll("(:)(.*?)(:)", ""); user = user.trim(); realurl = url[1]; realurl = realurl.replace("Visited:", ""); realurl = realurl.replaceAll(":(.*?):", ""); realurl = realurl.replace(":Host:", ""); realurl = realurl.trim(); } else { user = ""; realurl = lineBuff[1].trim(); } domain = Util.extractDomain(realurl); if (!ddtime.isEmpty()) { ddtime = ddtime.replace("T", " "); ddtime = ddtime.substring(ddtime.length() - 5); } if (!actime.isEmpty()) { try { Long epochtime = dateFormatter.parse(actime).getTime(); ftime = epochtime.longValue(); ftime = ftime / 1000; } catch (ParseException e) { logger.log(Level.SEVERE, "Error parsing Pasco results.", e); } } try { BlackboardArtifact bbart = tskCase.getContentById(artObjId).newArtifact(ARTIFACT_TYPE.TSK_WEB_HISTORY); Collection bbattributes = new ArrayList(); bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL.getTypeID(), "RecentActivity", realurl)); //bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL_DECODED.getTypeID(), "RecentActivity", EscapeUtil.decodeURL(realurl))); bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_ACCESSED.getTypeID(), "RecentActivity", ftime)); bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_REFERRER.getTypeID(), "RecentActivity", "")); // bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME.getTypeID(), "RecentActivity", ddtime)); bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PROG_NAME.getTypeID(), "RecentActivity", "Internet Explorer")); bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DOMAIN.getTypeID(), "RecentActivity", domain)); bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_USER_NAME.getTypeID(), "RecentActivity", user)); bbart.addAttributes(bbattributes); } catch (TskCoreException ex) { logger.log(Level.SEVERE, "Error writing Internet Explorer web history artifact to the blackboard.", ex); } } else { // @@@ Log that we didn't parse this } } } catch (FileNotFoundException ex) { logger.log(Level.WARNING, "Unable to find the Pasco file at " + file.getPath(), ex); } } } } } services.fireModuleDataEvent(new ModuleDataEvent("Recent Activity", BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_HISTORY)); } @Override public void init(IngestModuleInit initContext) { services = IngestServices.getDefault(); } @Override public void complete() { // Delete all the results when complete /*for (String file : pascoResults) { String filePath = moduleTempResultsDir + File.separator + file; try { File f = new File(filePath); if (f.exists() && f.canWrite()) { f.delete(); } else { logger.log(Level.WARNING, "Unable to delete file " + filePath); } } catch (SecurityException ex) { logger.log(Level.WARNING, "Incorrect permission to delete file " + filePath, ex); } } */ pascoResults.clear(); logger.info("Internet Explorer extract has completed."); } @Override public void stop() { if (execPasco != null) { execPasco.stop(); execPasco = null; } //call regular cleanup from complete() method complete(); } @Override public String getDescription() { return "Extracts activity from Internet Explorer browser, as well as recent documents in windows."; } @Override public boolean hasBackgroundJobsRunning() { return false; } }