Merge branch 'master' of https://github.com/sleuthkit/autopsy

Core/src/org/sleuthkit/autopsy/directorytree/ExtractUnallocAction.java

@@ -24,6 +24,7 @@ import java.awt.event.ActionEvent;
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
+import java.io.OutputStream;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
@@ -44,9 +45,9 @@ import org.sleuthkit.datamodel.ContentVisitor;
 import org.sleuthkit.datamodel.Directory;
 import org.sleuthkit.datamodel.FileSystem;
 import org.sleuthkit.datamodel.Image;
-import org.sleuthkit.datamodel.VirtualDirectory;
 import org.sleuthkit.datamodel.LayoutFile;
 import org.sleuthkit.datamodel.TskCoreException;
+import org.sleuthkit.datamodel.VirtualDirectory;
 import org.sleuthkit.datamodel.Volume;
 import org.sleuthkit.datamodel.VolumeSystem;
 
@@ -168,22 +169,23 @@ public final class ExtractUnallocAction extends AbstractAction {
         private List<UnallocStruct> lus = new ArrayList<UnallocStruct>();
         private File currentlyProcessing;
         private int totalSizeinMegs;
+        long totalBytes = 0;
 
         ExtractUnallocWorker(UnallocStruct us) {            
-            this.lus.add(us);
             //Getting the total megs this worker is going to be doing
             if (!lockedVols.contains(us.getFileName())) {
-                totalSizeinMegs = toMb(us.sizeInBytes());
+                this.lus.add(us);
+                totalBytes = us.getSizeInBytes();
+                totalSizeinMegs = toMb(totalBytes);
                 lockedVols.add(us.getFileName());
             }
         }
 
         ExtractUnallocWorker(List<UnallocStruct> lst) {
             //Getting the total megs this worker is going to be doing            
-            long totalBytes = 0;
             for (UnallocStruct lu : lst) {
                 if (!lockedVols.contains(lu.getFileName())) {
-                    totalBytes += lu.sizeInBytes();
+                    totalBytes += lu.getSizeInBytes();
                     lockedVols.add(lu.getFileName());
                     this.lus.add(lu);
                 }
@@ -196,7 +198,7 @@ public final class ExtractUnallocAction extends AbstractAction {
             if (bytes > 1024 && (bytes / 1024.0) <= Double.MAX_VALUE) {
                 double Mb = ((bytes / 1024.0) / 1024.0);//Bytes -> Megabytes
                 if (Mb <= Integer.MAX_VALUE) {
-                    return (int) Math.floor(Mb);
+                    return (int) Math.ceil(Mb);
                 }
             }
             return 0;
@@ -222,27 +224,32 @@ public final class ExtractUnallocAction extends AbstractAction {
 
                 //Begin the actual File IO
                 progress.start(totalSizeinMegs);
-                int kbs = 0; //Each completion of the while loop adds one to kbs. 8kb * 128 = 1mb. 
+                int kbs = 0; //Each completion of the while loop adds one to kbs. 16kb * 64 = 1mb. 
                 int mbs = 0; //Increments every 128th tick of  kbs
                 for (UnallocStruct u : this.lus) {
                     currentlyProcessing = u.getFile();
                     logger.log(Level.INFO, "Writing Unalloc file to " + currentlyProcessing.getPath());
-                    FileOutputStream fos = new FileOutputStream(currentlyProcessing);
+                    OutputStream dos = new FileOutputStream(currentlyProcessing);
-                    int count = 1;
+                    long bytes = 0;
-                    for (LayoutFile f : u.getLayouts()) {
+                    int i = 0;
-                        long offset = 0L;
+                    while(i < u.getLayouts().size() && bytes != u.getSizeInBytes()){                        
-                        while (offset != f.getSize() && !canceled) {
+                        LayoutFile f = u.getLayouts().get(i);
-                            offset += f.read(buf, offset, MAX_BYTES);    //Offset + Bytes read
+                        long offsetPerFile = 0L;
-                            fos.write(buf);
+                        int bytesRead;
+                        while(offsetPerFile != f.getSize() && !canceled){
                             if (++kbs % 128 == 0) {
                                 mbs++;                                
-                                progress.progress("processing " + mbs + " of " + totalSizeinMegs + " MBs", mbs);
+                                progress.progress("processing " + mbs + " of " + totalSizeinMegs + " MBs", mbs-1);
                             }
+                            bytesRead = f.read(buf, offsetPerFile, MAX_BYTES);  
+                            offsetPerFile+= bytesRead;
+                            dos.write(buf, 0, bytesRead);       
                         }
-                        count++;
+                        bytes+=f.getSize();
+                        i++;
                     }
-                    fos.flush();
+                    dos.flush();
-                    fos.close();
+                    dos.close();
 
                     if (canceled) {
                         u.getFile().delete();
@@ -420,9 +427,9 @@ public final class ExtractUnallocAction extends AbstractAction {
                 return 0;
             }
             if (o1.getId() > o2.getId()) {
-                return -1;
-            } else {
                 return 1;
+            } else {
+                return -1;
             }
         }
     }
@@ -434,6 +441,7 @@ public final class ExtractUnallocAction extends AbstractAction {
     private class UnallocStruct {
 
         private List<LayoutFile> llf;
+        private long SizeInBytes;
         private long VolumeId;
         private long ImageId;
         private String ImageName;
@@ -453,6 +461,7 @@ public final class ExtractUnallocAction extends AbstractAction {
             this.ImageName = img.getName();
             this.FileName = this.ImageName + "-Unalloc-" + this.ImageId + "-" + 0 + ".dat";
             this.FileInstance = new File(Case.getCurrentCase().getCaseDirectory() + File.separator + "Export" + File.separator + this.FileName);
+            this.SizeInBytes = calcSizeInBytes();
         }
 
         /**
@@ -474,6 +483,7 @@ public final class ExtractUnallocAction extends AbstractAction {
             this.FileInstance = new File(Case.getCurrentCase().getCaseDirectory() + File.separator + "Export" + File.separator + this.FileName);
             this.llf = getUnallocFiles(volu);
             Collections.sort(llf, new SortObjId());
+            this.SizeInBytes = calcSizeInBytes();
         }
 
         //Getters
@@ -481,7 +491,7 @@ public final class ExtractUnallocAction extends AbstractAction {
             return llf.size();
         }
 
-        long sizeInBytes() {
+       private long calcSizeInBytes() {
             long size = 0L;
             for (LayoutFile f : llf) {
                 size += f.getSize();
@@ -489,6 +499,10 @@ public final class ExtractUnallocAction extends AbstractAction {
             return size;
         }
        
+       long getSizeInBytes(){
+           return this.SizeInBytes;
+       }
+
         long getVolumeId() {
             return this.VolumeId;
         }

Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java

@@ -814,8 +814,7 @@ public class IngestManager {
 
                 final AbstractFile fileToProcess = fileTask.file;
                 
-                //logger.log(Level.INFO, "NEXT FILE: " + fileToProcess.getName());
+                logger.log(Level.INFO, "IngestManager: Processing: {0}", fileToProcess.getName());
-
                 progress.progress(fileToProcess.getName(), processedFiles);
 
                 for (IngestModuleAbstractFile module : fileTask.scheduledTask.modules) {
@@ -857,7 +856,7 @@ public class IngestManager {
                 //--totalEnqueuedFiles;
                 
             } //end of this AbstractFile
-            logger.log(Level.INFO, "Done background processing");
+            logger.log(Level.INFO, "IngestManager: Finished processing files");
             return null;
         }
 

Core/src/org/sleuthkit/autopsy/ingest/IngestScheduler.java

@@ -663,21 +663,31 @@ class IngestScheduler {
 
                 enum Priority {
 
-                    LOW, MEDIUM, HIGH
+                    LAST, LOW, MEDIUM, HIGH
                 };
+                static final List<Pattern> LAST_PRI_PATHS = new ArrayList<Pattern>();
                 static final List<Pattern> LOW_PRI_PATHS = new ArrayList<Pattern>();
                 static final List<Pattern> MEDIUM_PRI_PATHS = new ArrayList<Pattern>();
                 static final List<Pattern> HIGH_PRI_PATHS = new ArrayList<Pattern>();
 
+                /* prioritize root directory folders based on the assumption that we are
+                 * looking for user content. Other types of investigations may want different
+                 * priorities. */
                 static {
+                    // these files have no structure, so they go last
+                    LAST_PRI_PATHS.add(Pattern.compile("^\\$Unalloc", Pattern.CASE_INSENSITIVE));
+                    LAST_PRI_PATHS.add(Pattern.compile("^pagefile", Pattern.CASE_INSENSITIVE));
+                    LAST_PRI_PATHS.add(Pattern.compile("^hiberfil", Pattern.CASE_INSENSITIVE));
+                    
+                    // orphan files are often corrupt and windows does not typically have
+                    // user content, so put them towards the bottom
+                    LOW_PRI_PATHS.add(Pattern.compile("^\\$OrphanFiles", Pattern.CASE_INSENSITIVE));
                     LOW_PRI_PATHS.add(Pattern.compile("^Windows", Pattern.CASE_INSENSITIVE));
 
+                    // all other files go into the medium category too
                     MEDIUM_PRI_PATHS.add(Pattern.compile("^Program Files", Pattern.CASE_INSENSITIVE));
-                    MEDIUM_PRI_PATHS.add(Pattern.compile("^\\$OrphanFiles", Pattern.CASE_INSENSITIVE));
-                    MEDIUM_PRI_PATHS.add(Pattern.compile("^\\$Unalloc", Pattern.CASE_INSENSITIVE));
-                    MEDIUM_PRI_PATHS.add(Pattern.compile("^pagefile", Pattern.CASE_INSENSITIVE));
-                    MEDIUM_PRI_PATHS.add(Pattern.compile("^hiberfil", Pattern.CASE_INSENSITIVE));
 
+                    // user content is top priority
                     HIGH_PRI_PATHS.add(Pattern.compile("^Users", Pattern.CASE_INSENSITIVE));
                     HIGH_PRI_PATHS.add(Pattern.compile("^Documents and Settings", Pattern.CASE_INSENSITIVE));
                     HIGH_PRI_PATHS.add(Pattern.compile("^home", Pattern.CASE_INSENSITIVE));
@@ -685,10 +695,10 @@ class IngestScheduler {
                 }
 
                 static AbstractFilePriotity.Priority getPriority(final AbstractFile abstractFile) {
-                    if (!abstractFile.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.FS)) {
+                    //if (!abstractFile.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.FS)) {
                         //non-fs files, such as representing unalloc space
-                        return AbstractFilePriotity.Priority.MEDIUM;
+                        //return AbstractFilePriotity.Priority.MEDIUM;
-                    }
+                    //}
                     final String path = abstractFile.getName();
 
                     if (path == null) {
@@ -716,6 +726,13 @@ class IngestScheduler {
                         }
                     }
                     
+                    for (Pattern p : LAST_PRI_PATHS) {
+                        Matcher m = p.matcher(path);
+                        if (m.find()) {
+                            return AbstractFilePriotity.Priority.LAST;
+                        }
+                    }
+
                     //default is medium
                     return AbstractFilePriotity.Priority.MEDIUM;
                 }

Core/src/org/sleuthkit/autopsy/report/ReportHTML.java

@@ -755,7 +755,7 @@ public class ReportHTML implements ReportModule {
             out = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(folder + "downloads.html"), "UTF-8"));
             out.write(generateHead("Web Download Artifacts (" + countDownloads + ")"));
             String title = "<div id=\"header\">Web Downloads (" + countDownloads + ")</div>\n<div id=\"content\">\n";
-            String tableHeader = getTableHead("URL", "Source URL", "Date Accessed", "Program", "Source File");
+            String tableHeader = getTableHead("Destination", "Source URL", "Date Accessed", "Program", "Source File");
             out.write(title);
             out.write(tableHeader);
             

HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/IndexStatus.java

@@ -28,7 +28,7 @@ enum IndexStatus {
     /**
      * The index and database both exist, and the index is older.
      */
-    INDEX_OUTDATED("Index is older than database"),
+    INDEX_OUTDATED("WARNING: Index is older than database"),
     /**
      * The index and database both exist, and the index is not older.
      */
@@ -40,11 +40,11 @@ enum IndexStatus {
     /**
      * The database exists but the index does not.
      */
-    NO_INDEX("Index does not exist"),
+    NO_INDEX("ERROR: Index does not exist"),
     /**
      * Neither the index nor the database exists.
      */
-    NONE("No index or database"),
+    NONE("ERROR: No index or database"),
     /**
      * The index is currently being generated
      */

README.txt

@@ -47,7 +47,7 @@ JRE (Java Runtime Environment) 1.6, 32 bit
 - Web page: http://www.oracle.com/technetwork/java/index.html
 - License: http://www.oracle.com/technetwork/java/javase/terms/license/index.html
 
-Netbeans 7.0.1 RCP platform and .jar files bundled with the platform
+Netbeans 7.2.1 RCP platform and .jar files bundled with the platform
 - Web page: http://netbeans.org/features/platform/
 - License: 
 http://services.netbeans.org/downloads/licence/nb-7.0-final-2011-04-20-license.txt

RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Chrome.java

@@ -188,9 +188,33 @@ public class Chrome extends Extract implements IngestModuleImage {
                     for (JsonElement result : jBookmarkArray) {
                         try {
                             JsonObject address = result.getAsJsonObject();
-                            String url = address.get("url").getAsString();
+                            if (address == null) {
-                            String name = address.get("name").getAsString();
+                                continue;
-                            Long date = address.get("date_added").getAsLong();
+                            }
+                            JsonElement urlEl = address.get("url");
+                            String url = null;
+                            if (urlEl != null) {
+                                url = urlEl.getAsString();
+                            }
+                            else {
+                                url = "";
+                            }
+                            String name = null;
+                            JsonElement nameEl = address.get("name");
+                            if (nameEl != null) {
+                                name = nameEl.getAsString();
+                            }
+                            else {
+                                name = "";
+                            }
+                            Long date = null;
+                            JsonElement dateEl = address.get("date_added");
+                            if (dateEl != null) {
+                                date = dateEl.getAsLong();
+                            }
+                            else {
+                                date = Long.valueOf(0);
+                            }
                             String domain = Util.extractDomain(url);
                             BlackboardArtifact bbart = bookmarkFiles.get(j).newArtifact(ARTIFACT_TYPE.TSK_WEB_BOOKMARK);
                             Collection<BlackboardAttribute> bbattributes = new ArrayList<BlackboardAttribute>();

build.xml

@@ -7,7 +7,7 @@
 	
     <import file="nbproject/build-impl.xml"/>
     
-    <property name="netbeans-plat-version" value="7.2" />
+    <property name="netbeans-plat-version" value="7.2.1" />
     <property name="nbplatform.active.dir" value="${basedir}/netbeans-plat/${netbeans-plat-version}" />
 
     <condition property="os.family" value="unix">