diff --git a/Ingest/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Ingest/src/org/sleuthkit/autopsy/ingest/IngestManager.java index dd39b0001b..7f6081aa85 100755 --- a/Ingest/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Ingest/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -25,13 +25,12 @@ import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.Comparator; import java.util.Date; import java.util.HashMap; -import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.TreeMap; import java.util.concurrent.CancellationException; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; @@ -186,30 +185,30 @@ public class IngestManager { while (hasNextImage()) { //dequeue // get next image and set of services - final QueueUnit qu = + final Map.Entry> qu = this.getNextImage(); // check if each service for this image is already running //synchronized (this) { - for (IngestServiceImage quService : qu.services) { + for (IngestServiceImage quService : qu.getValue()) { boolean alreadyRunning = false; for (IngestImageThread worker : imageIngesters) { // ignore threads that are on different images - if (!worker.getImage().equals(qu.content)) { + if (!worker.getImage().equals(qu.getKey())) { continue; //check next worker } //same image, check service (by name, not id, since different instances) if (worker.getService().getName().equals(quService.getName())) { alreadyRunning = true; - logger.log(Level.INFO, "Image Ingester <" + qu.content + ", " + quService.getName() + "> is already running"); + logger.log(Level.INFO, "Image Ingester <" + qu.getKey() + ", " + quService.getName() + "> is already running"); break; } } //checked all workers if (alreadyRunning == false) { - logger.log(Level.INFO, "Starting new image Ingester <" + qu.content + ", " + quService.getName() + ">"); - IngestImageThread newImageWorker = new IngestImageThread(this, qu.content, quService); + logger.log(Level.INFO, "Starting new image Ingester <" + qu.getKey() + ", " + quService.getName() + ">"); + IngestImageThread newImageWorker = new IngestImageThread(this, qu.getKey(), quService); imageIngesters.add(newImageWorker); @@ -477,9 +476,7 @@ public class IngestManager { * @param service * @param image */ - private void addFsContent(IngestServiceFsContent service, Image image) { - Collection fsContents = new GetAllFilesContentVisitor().visit(image); - logger.log(Level.INFO, "Adding image " + image.getName() + " with " + fsContents.size() + " number of fsContent to service " + service.getName()); + private void addFsContent(IngestServiceFsContent service, Collection fsContents) { synchronized (queuesLock) { for (FsContent fsContent : fsContents) { fsContentQueue.enqueue(fsContent, service); @@ -492,8 +489,8 @@ public class IngestManager { * the queue of FsContent to process is maintained internally * and could be dynamically sorted as data comes in */ - private QueueUnit getNextFsContent() { - QueueUnit ret = null; + private Map.Entry> getNextFsContent() { + Map.Entry> ret = null; synchronized (queuesLock) { ret = fsContentQueue.dequeue(); } @@ -522,14 +519,6 @@ public class IngestManager { } } - private void sortFsContents() { - logger.log(Level.INFO, "Sorting fscontents"); - synchronized (queuesLock) { - fsContentQueue.sort(); - } - logger.log(Level.INFO, "Done sorting fscontents"); - } - private void emptyImages() { synchronized (queuesLock) { imageQueue.empty(); @@ -541,8 +530,8 @@ public class IngestManager { * the queue of Images to process is maintained internally * and could be dynamically sorted as data comes in */ - private QueueUnit getNextImage() { - QueueUnit ret = null; + private Map.Entry> getNextImage() { + Map.Entry> ret = null; synchronized (queuesLock) { ret = imageQueue.dequeue(); } @@ -643,48 +632,40 @@ public class IngestManager { */ private class FsContentQueue { - final List> fsContentUnits = new ArrayList>(); - final Comparator> sorter = new Comparator>() { + final Comparator sorter = new Comparator() { @Override - public int compare(QueueUnit q1, QueueUnit q2) { - FsContentPriotity.Priority p1 = FsContentPriotity.getPriority(q1.content); - FsContentPriotity.Priority p2 = FsContentPriotity.getPriority(q2.content); + public int compare(FsContent q1, FsContent q2) { + FsContentPriotity.Priority p1 = FsContentPriotity.getPriority(q1); + FsContentPriotity.Priority p2 = FsContentPriotity.getPriority(q2); if (p1 == p2) { - return (int) (q2.content.getId() - q1.content.getId()); + return (int) (q2.getId() - q1.getId()); } else { return p2.ordinal() - p1.ordinal(); } } }; + final TreeMap> fsContentUnits = new TreeMap>(sorter); void enqueue(FsContent fsContent, IngestServiceFsContent service) { - QueueUnit found = findFsContent(fsContent); - - if (found != null) { - //FsContent already enqueued - //merge services to use with already enqueued image - found.add(service); - } else { - //enqueue brand new FsContent with the services - found = new QueueUnit(fsContent, service); - fsContentUnits.add(found); + //fsContentUnits.put(fsContent, Collections.singletonList(service)); + List services = fsContentUnits.get(fsContent); + if(services == null) { + services = new ArrayList(); + fsContentUnits.put(fsContent, services); } + services.add(service); } void enqueue(FsContent fsContent, List services) { - QueueUnit found = findFsContent(fsContent); - if (found != null) { - //FsContent already enqueued - //merge services to use with already enqueued FsContent - found.addAll(services); - } else { - //enqueue brand new FsContent with the services - found = new QueueUnit(fsContent, services); - fsContentUnits.add(found); + List oldServices = fsContentUnits.get(fsContent); + if(oldServices == null) { + oldServices = new ArrayList(); + fsContentUnits.put(fsContent, oldServices); } + oldServices.addAll(services); } boolean hasNext() { @@ -699,22 +680,17 @@ public class IngestManager { fsContentUnits.clear(); } - void sort() { - Collections.sort(fsContentUnits, sorter); - } - /** * Returns next FsContent and list of associated services * @return */ - QueueUnit dequeue() { + Map.Entry> dequeue() { if (!hasNext()) { throw new UnsupportedOperationException("FsContent processing queue is empty"); } - QueueUnit remove = fsContentUnits.remove(0); //logger.log(Level.INFO, "DEQUE: " + remove.content.getParentPath() + " SIZE: " + toString()); - return (remove); + return (fsContentUnits.pollFirstEntry()); } /** @@ -723,30 +699,11 @@ public class IngestManager { * @return true if the service is enqueued to do work */ boolean hasServiceEnqueued(IngestServiceFsContent service) { - boolean found = false; - for (QueueUnit unit : fsContentUnits) { - for (IngestServiceFsContent s : unit.services) { - if (s.equals(service)) { - found = true; - break; - } - } - if (found == true) { - break; - } + for(List list : fsContentUnits.values()) { + if(list.contains(service)) + return true; } - return found; - } - - private QueueUnit findFsContent(FsContent fsContent) { - QueueUnit found = null; - for (QueueUnit unit : fsContentUnits) { - if (unit.content.equals(fsContent)) { - found = unit; - break; - } - } - return found; + return false; } @Override @@ -756,10 +713,10 @@ public class IngestManager { public String printQueue() { StringBuilder sb = new StringBuilder(); - for (QueueUnit u : fsContentUnits) { + /*for (QueueUnit u : fsContentUnits) { sb.append(u.toString()); sb.append("\n"); - } + }*/ return sb.toString(); } } @@ -771,34 +728,32 @@ public class IngestManager { */ private class ImageQueue { - private List> imageUnits = new ArrayList>(); + final Comparator sorter = new Comparator() { + + @Override + public int compare(Image q1, Image q2) { + return (int) (q2.getId() - q1.getId()); + + } + }; + private TreeMap> imageUnits = new TreeMap>(sorter); void enqueue(Image image, IngestServiceImage service) { - QueueUnit found = findImage(image); - - if (found != null) { - //image already enqueued - //merge services to use with already enqueued image - found.add(service); - } else { - //enqueue brand new image with the services - found = new QueueUnit(image, service); - imageUnits.add(found); + List services = imageUnits.get(image); + if(services == null) { + services = new ArrayList(); + imageUnits.put(image, services); } + services.add(service); } void enqueue(Image image, List services) { - QueueUnit found = findImage(image); - - if (found != null) { - //image already enqueued - //merge services to use with already enqueued image - found.addAll(services); - } else { - //enqueue brand new image with the services - found = new QueueUnit(image, services); - imageUnits.add(found); + List oldServices = imageUnits.get(image); + if(oldServices == null) { + oldServices = new ArrayList(); + imageUnits.put(image, oldServices); } + oldServices.addAll(services); } boolean hasNext() { @@ -817,29 +772,12 @@ public class IngestManager { * Return a QueueUnit that contains an image and set of services to run on it. * @return */ - QueueUnit dequeue() { + Map.Entry> dequeue() { if (!hasNext()) { throw new UnsupportedOperationException("Image processing queue is empty"); } - return imageUnits.remove(0); - } - - /** - * Search existing list to see if an image already has a set of - * services associated with it - * @param image - * @return - */ - private QueueUnit findImage(Image image) { - QueueUnit found = null; - for (QueueUnit unit : imageUnits) { - if (unit.content.equals(image)) { - found = unit; - break; - } - } - return found; + return imageUnits.pollFirstEntry(); } @Override @@ -848,79 +786,6 @@ public class IngestManager { } } - /** - * generic representation of queued content (Image or FsContent) and its services - */ - private class QueueUnit { - - T content; - LinkedHashSet services; //ordering matters (Lookup order) - - QueueUnit(T content, S service) { - this.content = content; - this.services = new LinkedHashSet(); - add(service); - } - - QueueUnit(T content, List services) { - this.content = content; - this.services = new LinkedHashSet(); - addAll(services); - } - - //merge services with the current collection of services per image - //this assumes singleton instances of every service type for correct merge - //in case of multiple instances, they need to be handled correctly after dequeue() - final void addAll(List services) { - this.services.addAll(services); - } - - //this assumes singleton instances of every service type for correct merge - //in case of multiple instances, they need to be handled correctly after dequeue() - final void add(S service) { - this.services.add(service); - } - - @Override - @SuppressWarnings("unchecked") - public boolean equals(Object obj) { - if (obj == null) { - return false; - } - if (getClass() != obj.getClass()) { - return false; - } - final QueueUnit other = (QueueUnit) obj; - if (this.content != other.content && (this.content == null || !this.content.equals(other.content))) { - return false; - } - if (this.services != other.services && (this.services == null || !this.services.equals(other.services))) { - return false; - } - return true; - } - - @Override - public int hashCode() { - int hash = 7; - hash = 37 * hash + (this.content != null ? this.content.hashCode() : 0); - hash = 37 * hash + (this.services != null ? this.services.hashCode() : 0); - return hash; - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append("content: "); - sb.append(content.toString()); - sb.append("\nservices: "); - for (S service : services) { - sb.append(service.toString()).append(" "); - } - return sb.toString(); - } - } - /** * collects IngestManager statistics during runtime */ @@ -1065,19 +930,19 @@ public class IngestManager { initMainProgress(numFsContents); //process fscontents queue while (hasNextFsContent()) { - QueueUnit unit = getNextFsContent(); + Map.Entry> unit = getNextFsContent(); //clear return values from services for last file synchronized (fsContentServiceResults) { fsContentServiceResults.clear(); } - for (IngestServiceFsContent service : unit.services) { + for (IngestServiceFsContent service : unit.getValue()) { if (isCancelled()) { return null; } try { - IngestServiceFsContent.ProcessResult result = service.process(unit.content); + IngestServiceFsContent.ProcessResult result = service.process(unit.getKey()); //handle unconditional stop if (result == IngestServiceFsContent.ProcessResult.STOP) { break; @@ -1103,7 +968,7 @@ public class IngestManager { initMainProgress(numFsContents); } - progress.progress(unit.content.getName(), ++processedFiles); + progress.progress(unit.getKey().getName(), ++processedFiles); --numFsContents; } //end of this fsContent logger.log(Level.INFO, "Done background processing"); @@ -1270,6 +1135,7 @@ public class IngestManager { int processed = 0; for (Image image : images) { final String imageName = image.getName(); + Collection fsContents = null; for (IngestServiceAbstract service : services) { if (isCancelled()) { return; @@ -1292,20 +1158,27 @@ public class IngestManager { //addImage((IngestServiceImage) service, image); break; case FsContent: + if(fsContents == null) { + long start = System.currentTimeMillis(); + fsContents = new GetAllFilesContentVisitor().visit(image); + logger.info("Get all files took " + (System.currentTimeMillis()-start) + "ms"); + } //enqueue the same singleton fscontent service - addFsContent((IngestServiceFsContent) service, image); + logger.log(Level.INFO, "Adding image " + image.getName() + " with " + fsContents.size() + " number of fsContent to service " + service.getName()); + addFsContent((IngestServiceFsContent) service, fsContents); break; default: logger.log(Level.SEVERE, "Unexpected service type: " + service.getType().name()); } progress.progress(serviceName + " " + imageName, ++processed); } + if(fsContents != null) + fsContents.clear(); } //logger.log(Level.INFO, fsContentQueue.printQueue()); progress.progress("Sorting files", processed); - sortFsContents(); } private void handleInterruption() {