mstoeck3/autopsy-flatpak
1
0
Fork 0
mirror of https://github.com/overcuriousity/autopsy-flatpak.git synced 2025-07-06 21:00:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Basic runnable ingest framework.

Browse Source
This commit is contained in:
adam-m 2012-01-25 17:33:08 -05:00
parent 8b72034284
commit 9b460ac359
14 changed files with 622 additions and 140 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Case/nbproject/genfiles.properties
View File

@ -1,8 +1,8 @@
build.xml.data.CRC32=2b495fd9
build.xml.data.CRC32=a2330d9e
build.xml.script.CRC32=601bc2ba
build.xml.stylesheet.CRC32=a56c6a5b@1.46.2
# This file is used by a NetBeans-based IDE to track changes in generated files such as build-impl.xml.
# Do not edit this file. You may delete it but then the IDE will never regenerate such files for you.
nbproject/build-impl.xml.data.CRC32=2b495fd9
nbproject/build-impl.xml.data.CRC32=a2330d9e
nbproject/build-impl.xml.script.CRC32=65e93a36
nbproject/build-impl.xml.stylesheet.CRC32=238281d1@1.46.2

9
Case/nbproject/project.xml
View File

@ -123,15 +123,6 @@
<specification-version>1.0</specification-version>
</run-dependency>
</dependency>
<dependency>
<code-name-base>org.sleuthkit.autopsy.ingest</code-name-base>
<build-prerequisite/>
<compile-dependency/>
<run-dependency>
<release-version>0-1</release-version>
<specification-version>1.0</specification-version>
</run-dependency>
</dependency>
</module-dependencies>
<test-dependencies>
<test-type>

4
Ingest/nbproject/genfiles.properties
View File

@ -1,8 +1,8 @@
build.xml.data.CRC32=1833dad1
build.xml.data.CRC32=41cda326
build.xml.script.CRC32=7d5a04db
build.xml.stylesheet.CRC32=a56c6a5b@1.46.2
# This file is used by a NetBeans-based IDE to track changes in generated files such as build-impl.xml.
# Do not edit this file. You may delete it but then the IDE will never regenerate such files for you.
nbproject/build-impl.xml.data.CRC32=1833dad1
nbproject/build-impl.xml.data.CRC32=41cda326
nbproject/build-impl.xml.script.CRC32=8b2da6d1
nbproject/build-impl.xml.stylesheet.CRC32=238281d1@1.46.2

9
Ingest/nbproject/project.xml
View File

@ -85,6 +85,15 @@
<specification-version>6.40.1</specification-version>
</run-dependency>
</dependency>
<dependency>
<code-name-base>org.sleuthkit.autopsy.casemodule</code-name-base>
<build-prerequisite/>
<compile-dependency/>
<run-dependency>
<release-version>1</release-version>
<specification-version>1.0</specification-version>
</run-dependency>
</dependency>
<dependency>
<code-name-base>org.sleuthkit.autopsy.corecomponentinterfaces</code-name-base>
<build-prerequisite/>

2
Ingest/src/org/sleuthkit/autopsy/ingest/Bundle.properties
View File

@ -5,5 +5,3 @@ OpenIDE-Module-Name=Ingest
IngestTopComponent.topLable.text=Image ingest services
IngestTopComponent.startButton.text=Start
IngestTopComponent.refreshFreqLabel.text=Refresh frequency
IngestTopComponent.fileProgressLabel.text=File progress
IngestTopComponent.imageProgressLabel.text=Image progress

67
Ingest/src/org/sleuthkit/autopsy/ingest/GetAllFilesContentVisitor.java Normal file
View File

@ -0,0 +1,67 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2011 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.ingest;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.Collection;
import java.util.Collections;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.datamodel.File;
import org.sleuthkit.datamodel.FileSystem;
import org.sleuthkit.datamodel.FsContent;
import org.sleuthkit.datamodel.SleuthkitCase;
import org.sleuthkit.datamodel.TskData;
import org.sleuthkit.datamodel.TskData.FileKnown;
/**
* Visitor for getting all the files to try to index from any Content object.
* Currently gets all non-zero files.
* TODO should be moved to utility module (needs resolve cyclic deps)
*/
class GetAllFilesContentVisitor extends GetFilesContentVisitor {
private static final Logger logger = Logger.getLogger(GetAllFilesContentVisitor.class.getName());
@Override
public Collection<FsContent> visit(File file) {
return Collections.singleton((FsContent) file);
}
@Override
public Collection<FsContent> visit(FileSystem fs) {
// Files in the database have a filesystem field, so it's quick to
// get all the matching files for an entire filesystem with a query
SleuthkitCase sc = Case.getCurrentCase().getSleuthkitCase();
String query = "SELECT * FROM tsk_files WHERE fs_obj_id = " + fs.getId()
+ " AND (meta_type = " + TskData.TSK_FS_META_TYPE_ENUM.TSK_FS_META_TYPE_REG.getMetaType() +
") AND (known != " + FileKnown.KNOWN.toLong() + ") AND (size > 0)";
try {
ResultSet rs = sc.runQuery(query);
return sc.resultSetToFsContents(rs);
} catch (SQLException ex) {
logger.log(Level.WARNING, "Couldn't get all files in FileSystem", ex);
return Collections.EMPTY_SET;
}
}
}

105
Ingest/src/org/sleuthkit/autopsy/ingest/GetFilesContentVisitor.java Normal file
View File

@ -0,0 +1,105 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2011 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.ingest;
import java.util.ArrayList;
import java.util.Collection;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.sleuthkit.datamodel.Content;
import org.sleuthkit.datamodel.ContentVisitor;
import org.sleuthkit.datamodel.Directory;
import org.sleuthkit.datamodel.File;
import org.sleuthkit.datamodel.FileSystem;
import org.sleuthkit.datamodel.FsContent;
import org.sleuthkit.datamodel.Image;
import org.sleuthkit.datamodel.TskException;
import org.sleuthkit.datamodel.Volume;
import org.sleuthkit.datamodel.VolumeSystem;
/**
* Abstract visitor for getting all the files from content
* TODO should be moved to utility module (needs resolve cyclic deps)
*/
public abstract class GetFilesContentVisitor implements ContentVisitor<Collection<FsContent>> {
private static final Logger logger = Logger.getLogger(GetFilesContentVisitor.class.getName());
@Override
public abstract Collection<FsContent> visit(File file);
@Override
public abstract Collection<FsContent> visit(FileSystem fs);
@Override
public Collection<FsContent> visit(Directory drctr) {
return getAllFromChildren(drctr);
}
@Override
public Collection<FsContent> visit(Image image) {
return getAllFromChildren(image);
}
@Override
public Collection<FsContent> visit(Volume volume) {
return getAllFromChildren(volume);
}
@Override
public Collection<FsContent> visit(VolumeSystem vs) {
return getAllFromChildren(vs);
}
/**
* Aggregate all the matches from visiting the children Content objects of the
* one passed
* @param parent
* @return
*/
protected Collection<FsContent> getAllFromChildren(Content parent) {
Collection<FsContent> all = new ArrayList<FsContent>();
try {
for (Content child : parent.getChildren()) {
all.addAll(child.accept(this));
}
} catch (TskException ex) {
logger.log(Level.SEVERE, "Error getting Content children", ex);
}
return all;
}
/**
* Get the part of a file name after (not including) the last '.' and
* coerced to lowercase.
* @param fileName
* @return the file extension, or an empty string if there is none
*/
protected static String getExtension(String fileName) {
int lastDot = fileName.lastIndexOf(".");
if (lastDot >= 0) {
return fileName.substring(lastDot + 1, fileName.length()).toLowerCase();
} else {
return "";
}
}
}

396
Ingest/src/org/sleuthkit/autopsy/ingest/IngestManager.java
View File

@ -20,12 +20,19 @@ package org.sleuthkit.autopsy.ingest;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.swing.SwingUtilities;
import javax.swing.SwingWorker;
import org.openide.util.Lookup;
import org.sleuthkit.datamodel.FsContent;
import org.sleuthkit.datamodel.Image;
@ -38,13 +45,18 @@ import org.sleuthkit.datamodel.Image;
*
*/
public class IngestManager {
private static final Logger logger = Logger.getLogger(IngestManager.class.getName());
private IngestTopComponent tc;
private IngestManagerStats stats;
private int updateFrequency;
//queues
private final Object queueLock = new Object();
private final ImageQueue imageQueue = new ImageQueue();
private final FsContentQueue fsContentQueue = new FsContentQueue();
private IngestThread ingester;
final Collection<IngestServiceImage> imageServices = enumerateImageServices();
final Collection<IngestServiceFsContent> fsContentServices = enumerateFsContentServices();
/**
*
@ -52,7 +64,14 @@ public class IngestManager {
*/
IngestManager(IngestTopComponent tc) {
this.tc = tc;
stats = new IngestManagerStats();
//one time initialization of services
for (IngestServiceImage s : imageServices) {
s.init(this);
}
for (IngestServiceFsContent s : fsContentServices) {
s.init(this);
}
}
/**
@ -61,9 +80,46 @@ public class IngestManager {
* Notifies services when work is complete or should be interrupted using complete() and stop() calls.
* Does not block and can be called multiple times to enqueue more work to already running background process.
*/
void execute(Collection<IngestServiceAbstract> services, Image image) {
void execute(Collection<IngestServiceAbstract> services, final Collection<Image> images) {
for (Image image : images) {
for (IngestServiceAbstract service : services) {
switch (service.getType()) {
case Image:
addImage((IngestServiceImage) service, image);
break;
case FsContent:
addFsContent((IngestServiceFsContent) service, image);
break;
default:
logger.log(Level.SEVERE, "Unexpected service type: " + service.getType().name());
}
}
}
logger.log(Level.INFO, "Queues: " + imageQueue.toString() + " " + fsContentQueue.toString());
boolean start = false;
if (ingester == null) {
start = true;
} //if worker had completed, restart it in case data is still enqueued
else if (ingester.isDone()
&& (hasNextFsContent() || hasNextImage())) {
logger.log(Level.INFO, "Restarting ingester thread.");
start = true;
} else {
logger.log(Level.INFO, "Ingester is still running");
}
if (start) {
logger.log(Level.INFO, "Starting new ingester.");
ingester = new IngestThread();
stats = new IngestManagerStats();
ingester.execute();
}
}
/**
* returns the current minimal update frequency setting
* Services should call this between processing iterations to get current setting
@ -72,7 +128,7 @@ public class IngestManager {
public synchronized int getUpdateFrequency() {
return updateFrequency;
}
/**
* set new minimal update frequency services should use
* @param frequency
@ -96,9 +152,9 @@ public class IngestManager {
* Viewer will attempt to identify duplicate messages and filter them out (slower)
*/
public synchronized void postMessage(final IngestMessage message) {
SwingUtilities.invokeLater(new Runnable() {
@Override
public void run() {
tc.displayMessage(message);
@ -120,17 +176,47 @@ public class IngestManager {
return (Collection<IngestServiceFsContent>) Lookup.getDefault().lookupAll(IngestServiceFsContent.class);
}
private void addImage(IngestServiceImage service, Image image) {
synchronized (queueLock) {
imageQueue.enqueue(image, service);
//queueLock.notifyAll();
}
}
private void addFsContent(IngestServiceFsContent service, Image image) {
Collection<FsContent> fsContents = new GetAllFilesContentVisitor().visit(image);
synchronized (queueLock) {
for (FsContent fsContent : fsContents) {
fsContentQueue.enqueue(fsContent, service);
}
//queueLock.notifyAll();
}
//logger.log(Level.INFO, fsContentQueue.toString());
}
/**
* get next file/dir to process
* the queue of FsContent to process is maintained internally
* and could be dynamically sorted as data comes in
*/
private synchronized FsContent getNextFsContent() {
return null;
private QueueUnit<FsContent, IngestServiceFsContent> getNextFsContent() {
QueueUnit<FsContent, IngestServiceFsContent> ret = null;
synchronized (queueLock) {
ret = fsContentQueue.dequeue();
}
return ret;
}
private synchronized boolean hasNextFsContent() {
return true;
private boolean hasNextFsContent() {
boolean ret = false;
synchronized (queueLock) {
ret = fsContentQueue.hasNext();
}
return ret;
}
/**
@ -138,12 +224,176 @@ public class IngestManager {
* the queue of Images to process is maintained internally
* and could be dynamically sorted as data comes in
*/
private synchronized Image getNextImage() {
return null;
private QueueUnit<Image, IngestServiceImage> getNextImage() {
QueueUnit<Image, IngestServiceImage> ret = null;
synchronized (queueLock) {
ret = imageQueue.dequeue();
}
return ret;
}
private synchronized boolean hasNextImage() {
return true;
private boolean hasNextImage() {
boolean ret = false;
synchronized (queueLock) {
ret = imageQueue.hasNext();
}
return ret;
}
//manages queue of pending FsContent and IngestServiceFsContent to use on that content
//TODO in future content sort will be maintained based on priorities
private class FsContentQueue {
List<QueueUnit<FsContent, IngestServiceFsContent>> fsContentUnits = new ArrayList<QueueUnit<FsContent, IngestServiceFsContent>>();
void enqueue(FsContent fsContent, IngestServiceFsContent service) {
QueueUnit<FsContent, IngestServiceFsContent> found = findFsContent(fsContent);
if (found != null) {
//FsContent already enqueued
//merge services to use with already enqueued image
found.add(service);
} else {
//enqueue brand new FsContent with the services
found = new QueueUnit<FsContent, IngestServiceFsContent>(fsContent, service);
fsContentUnits.add(found);
}
}
void enqueue(FsContent fsContent, Collection<IngestServiceFsContent> services) {
QueueUnit<FsContent, IngestServiceFsContent> found = findFsContent(fsContent);
if (found != null) {
//FsContent already enqueued
//merge services to use with already enqueued FsContent
found.addAll(services);
} else {
//enqueue brand new FsContent with the services
found = new QueueUnit<FsContent, IngestServiceFsContent>(fsContent, services);
fsContentUnits.add(found);
}
}
boolean hasNext() {
return !fsContentUnits.isEmpty();
}
QueueUnit<FsContent, IngestServiceFsContent> dequeue() {
if (!hasNext()) {
throw new UnsupportedOperationException("FsContent processing queue is empty");
}
return fsContentUnits.remove(0);
}
private QueueUnit<FsContent, IngestServiceFsContent> findFsContent(FsContent fsContent) {
QueueUnit<FsContent, IngestServiceFsContent> found = null;
for (QueueUnit<FsContent, IngestServiceFsContent> unit : fsContentUnits) {
if (unit.content.equals(fsContent)) {
found = unit;
break;
}
}
return found;
}
@Override
public String toString() {
return "FsContentQueue, size: " + Integer.toString(fsContentUnits.size());
}
}
//manages queue of pending Images and IngestServiceImage to use on that image
private class ImageQueue {
List<QueueUnit<Image, IngestServiceImage>> imageUnits = new ArrayList<QueueUnit<Image, IngestServiceImage>>();
void enqueue(Image image, IngestServiceImage service) {
QueueUnit<Image, IngestServiceImage> found = findImage(image);
if (found != null) {
//image already enqueued
//merge services to use with already enqueued image
found.add(service);
} else {
//enqueue brand new image with the services
found = new QueueUnit<Image, IngestServiceImage>(image, service);
imageUnits.add(found);
}
}
void enqueue(Image image, Collection<IngestServiceImage> services) {
QueueUnit<Image, IngestServiceImage> found = findImage(image);
if (found != null) {
//image already enqueued
//merge services to use with already enqueued image
found.addAll(services);
} else {
//enqueue brand new image with the services
found = new QueueUnit<Image, IngestServiceImage>(image, services);
imageUnits.add(found);
}
}
boolean hasNext() {
return !imageUnits.isEmpty();
}
QueueUnit<Image, IngestServiceImage> dequeue() {
if (!hasNext()) {
throw new UnsupportedOperationException("Image processing queue is empty");
}
return imageUnits.remove(0);
}
private QueueUnit<Image, IngestServiceImage> findImage(Image image) {
QueueUnit<Image, IngestServiceImage> found = null;
for (QueueUnit<Image, IngestServiceImage> unit : imageUnits) {
if (unit.content.equals(image)) {
found = unit;
break;
}
}
return found;
}
@Override
public String toString() {
return "ImageQueue, size: " + Integer.toString(imageUnits.size());
}
}
/**
* generic representation of queued content (Image or FsContent) and its services
*/
private class QueueUnit<T, S> {
T content;
Set<S> services;
QueueUnit(T content, S service) {
this.content = content;
this.services = new HashSet<S>();
add(service);
}
QueueUnit(T content, Collection<S> services) {
this.content = content;
this.services = new HashSet<S>();
addAll(services);
}
//merge services with the current collection of services per image
//this assumes that there is one singleton instance of each type of service
final void addAll(Collection<S> services) {
this.services.addAll(services);
}
final void add(S service) {
this.services.add(service);
}
}
/**
@ -156,32 +406,32 @@ public class IngestManager {
int errorsTotal;
Map<IngestServiceAbstract, Integer> errors;
private static DateFormat dateFormatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
IngestManagerStats() {
errors = new HashMap<IngestServiceAbstract, Integer>();
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
if (startTime != null) {
sb.append("Start time: ").append(dateFormatter.format(startTime));
sb.append("Start time: ").append(dateFormatter.format(startTime)).append("\n");
}
if (endTime != null) {
sb.append("End time: ").append(dateFormatter.format(endTime));
sb.append("End time: ").append(dateFormatter.format(endTime)).append("\n");
}
sb.append("Total ingest time: ").append(getTotalTime());
sb.append("Total errors: ").append(errorsTotal);
sb.append("Total ingest time: ").append(getTotalTime()).append("\n");
sb.append("Total errors: ").append(errorsTotal).append("\n");
if (errorsTotal > 0) {
sb.append("Errors per service:");
for (IngestServiceAbstract service : errors.keySet()) {
final int errorsService = errors.get(service);
sb.append("\t").append(service.getName()).append(": ").append(errorsService);
sb.append("\t").append(service.getName()).append(": ").append(errorsService).append("\n");
}
}
return sb.toString();
}
void start() {
startTime = new Date();
}
@ -189,18 +439,106 @@ public class IngestManager {
void end() {
endTime = new Date();
}
long getTotalTime() {
if (startTime == null || endTime == null) {
return 0;
}
return endTime.getTime() - startTime.getTime();
}
void addError(IngestServiceAbstract source) {
++errorsTotal;
int curServiceError = errors.get(source);
errors.put(source, curServiceError + 1);
}
}
//ingester worker doing work in background
//in current design, worker runs until queues are consumed
//and if needed, it is restarted when data arrives
private class IngestThread extends SwingWorker {
private Logger logger = Logger.getLogger(IngestThread.class.getName());
@Override
protected Object doInBackground() throws Exception {
logger.log(Level.INFO, "Starting background processing");
stats.start();
//process image queue
while (hasNextImage()) {
QueueUnit<Image, IngestServiceImage> unit = getNextImage();
for (IngestServiceImage service : unit.services) {
if (isCancelled()) {
for (IngestServiceImage s : imageServices) {
s.stop();
}
return null;
}
try {
service.process(unit.content);
} catch (Exception e) {
logger.log(Level.INFO, "Exception from service: " + service.getName(), e);
stats.addError(service);
}
}
}
//process fscontents queue
while (hasNextFsContent()) {
QueueUnit<FsContent, IngestServiceFsContent> unit = getNextFsContent();
for (IngestServiceFsContent service : unit.services) {
if (isCancelled()) {
for (IngestServiceFsContent s : fsContentServices) {
s.stop();
}
return null;
}
try {
service.process(unit.content);
} catch (Exception e) {
logger.log(Level.INFO, "Exception from service: " + service.getName(), e);
stats.addError(service);
}
}
}
stats.end();
logger.log(Level.INFO, "Done background processing");
return null;
}
@Override
protected void done() {
try {
super.get(); //block and get all exceptions thrown while doInBackground()
logger.log(Level.INFO, "STATS: " + stats.toString());
//notify services of completion
for (IngestServiceImage s : imageServices) {
s.complete();
}
for (IngestServiceFsContent s : fsContentServices) {
s.complete();
}
} catch (InterruptedException ex) {
} catch (ExecutionException ex) {
logger.log(Level.SEVERE, "Fatal error during ingest.", ex);
} catch (Exception ex) {
logger.log(Level.SEVERE, "Fatal error during ingest.", ex);
}
}
@Override
protected void process(List chunks) {
super.process(chunks);
}
}
}

6
Ingest/src/org/sleuthkit/autopsy/ingest/IngestMessage.java
View File

@ -72,10 +72,10 @@ public class IngestMessage {
StringBuilder sb = new StringBuilder();
sb.append(Long.toString(ID)).append(": ");
sb.append("type: ").append(messageType.name());
sb.append("source: ").append(source.getName());
sb.append("text: ").append(text);
sb.append(" source: ").append(source.getName());
sb.append(" text: ").append(text);
if (data != null)
sb.append("data: ").append(data.toString());
sb.append(" data: ").append(data.toString()).append(' ');
return sb.toString();
}

38
Ingest/src/org/sleuthkit/autopsy/ingest/IngestTopComponent.form
View File

@ -47,8 +47,6 @@
<Group type="103" groupAlignment="1" attributes="0">
<Component id="topLable" alignment="0" min="-2" max="-2" attributes="1"/>
<Group type="103" alignment="0" groupAlignment="1" max="-2" attributes="0">
<Component id="jProgressBar1" alignment="0" max="32767" attributes="0"/>
<Component id="imageProgressBar" alignment="0" max="32767" attributes="0"/>
<Component id="servicesPanel" alignment="0" max="32767" attributes="1"/>
<Component id="freqSlider" alignment="0" max="32767" attributes="1"/>
</Group>
@ -58,18 +56,10 @@
<EmptySpace max="-2" attributes="0"/>
<Component id="startButton" min="-2" max="-2" attributes="0"/>
</Group>
<Group type="102" alignment="0" attributes="0">
<EmptySpace min="-2" pref="81" max="-2" attributes="0"/>
<Component id="imageProgressLabel" min="-2" max="-2" attributes="0"/>
</Group>
<Group type="102" alignment="0" attributes="0">
<EmptySpace min="-2" pref="74" max="-2" attributes="0"/>
<Component id="refreshFreqLabel" min="-2" max="-2" attributes="0"/>
</Group>
<Group type="102" alignment="0" attributes="0">
<EmptySpace min="-2" pref="84" max="-2" attributes="0"/>
<Component id="fileProgressLabel" min="-2" max="-2" attributes="0"/>
</Group>
</Group>
<EmptySpace pref="173" max="32767" attributes="0"/>
</Group>
@ -88,15 +78,7 @@
<Component id="freqSlider" min="-2" max="-2" attributes="0"/>
<EmptySpace max="-2" attributes="0"/>
<Component id="refreshFreqLabel" min="-2" max="-2" attributes="0"/>
<EmptySpace min="-2" pref="28" max="-2" attributes="0"/>
<Component id="imageProgressBar" min="-2" max="-2" attributes="0"/>
<EmptySpace max="-2" attributes="0"/>
<Component id="imageProgressLabel" min="-2" max="-2" attributes="0"/>
<EmptySpace min="-2" pref="27" max="-2" attributes="0"/>
<Component id="jProgressBar1" min="-2" max="-2" attributes="0"/>
<EmptySpace max="-2" attributes="0"/>
<Component id="fileProgressLabel" min="-2" max="-2" attributes="0"/>
<EmptySpace pref="75" max="32767" attributes="0"/>
<EmptySpace pref="198" max="32767" attributes="0"/>
</Group>
</Group>
</DimensionLayout>
@ -170,24 +152,6 @@
</Property>
</Properties>
</Component>
<Component class="javax.swing.JProgressBar" name="imageProgressBar">
</Component>
<Component class="javax.swing.JLabel" name="imageProgressLabel">
<Properties>
<Property name="text" type="java.lang.String" editor="org.netbeans.modules.i18n.form.FormI18nStringEditor">
<ResourceString bundle="org/sleuthkit/autopsy/ingest/Bundle.properties" key="IngestTopComponent.imageProgressLabel.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
</Property>
</Properties>
</Component>
<Component class="javax.swing.JProgressBar" name="jProgressBar1">
</Component>
<Component class="javax.swing.JLabel" name="fileProgressLabel">
<Properties>
<Property name="text" type="java.lang.String" editor="org.netbeans.modules.i18n.form.FormI18nStringEditor">
<ResourceString bundle="org/sleuthkit/autopsy/ingest/Bundle.properties" key="IngestTopComponent.fileProgressLabel.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
</Property>
</Properties>
</Component>
</SubComponents>
</Container>
</SubComponents>

103
Ingest/src/org/sleuthkit/autopsy/ingest/IngestTopComponent.java
View File

@ -22,6 +22,7 @@ import java.awt.BorderLayout;
import java.awt.event.ActionEvent;
import java.awt.event.ActionListener;
import java.beans.PropertyChangeEvent;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
@ -35,7 +36,11 @@ import javax.swing.JScrollPane;
import javax.swing.JSlider;
import org.openide.util.NbBundle;
import org.openide.windows.TopComponent;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.corecomponentinterfaces.DataExplorer;
import org.sleuthkit.datamodel.Image;
import org.sleuthkit.datamodel.SleuthkitCase;
import org.sleuthkit.datamodel.TskException;
/**
* Top component explorer for the Ingest module.
@ -57,7 +62,6 @@ public final class IngestTopComponent extends TopComponent implements DataExplor
};
private IngestTopComponent() {
manager = new IngestManager(this);
services = new ArrayList<IngestServiceAbstract>();
serviceStates = new HashMap<String, Boolean>();
initComponents();
@ -139,14 +143,10 @@ public final class IngestTopComponent extends TopComponent implements DataExplor
freqSlider = new javax.swing.JSlider();
startButton = new javax.swing.JButton();
refreshFreqLabel = new javax.swing.JLabel();
imageProgressBar = new javax.swing.JProgressBar();
imageProgressLabel = new javax.swing.JLabel();
jProgressBar1 = new javax.swing.JProgressBar();
fileProgressLabel = new javax.swing.JLabel();
mainScrollPane.setPreferredSize(new java.awt.Dimension(289, 509));
topLable.setFont(new java.awt.Font("Tahoma", 0, 12)); // NOI18N
topLable.setFont(new java.awt.Font("Tahoma", 0, 12));
org.openide.awt.Mnemonics.setLocalizedText(topLable, org.openide.util.NbBundle.getMessage(IngestTopComponent.class, "IngestTopComponent.topLable.text")); // NOI18N
servicesPanel.setBorder(javax.swing.BorderFactory.createLineBorder(new java.awt.Color(0, 0, 0)));
@ -185,43 +185,26 @@ public final class IngestTopComponent extends TopComponent implements DataExplor
org.openide.awt.Mnemonics.setLocalizedText(refreshFreqLabel, org.openide.util.NbBundle.getMessage(IngestTopComponent.class, "IngestTopComponent.refreshFreqLabel.text")); // NOI18N
org.openide.awt.Mnemonics.setLocalizedText(imageProgressLabel, org.openide.util.NbBundle.getMessage(IngestTopComponent.class, "IngestTopComponent.imageProgressLabel.text")); // NOI18N
org.openide.awt.Mnemonics.setLocalizedText(fileProgressLabel, org.openide.util.NbBundle.getMessage(IngestTopComponent.class, "IngestTopComponent.fileProgressLabel.text")); // NOI18N
javax.swing.GroupLayout mainPanelLayout = new javax.swing.GroupLayout(mainPanel);
mainPanel.setLayout(mainPanelLayout);
mainPanelLayout.setHorizontalGroup(
mainPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addGroup(mainPanelLayout.createSequentialGroup()
.addGroup(mainPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.TRAILING)
.addGroup(javax.swing.GroupLayout.Alignment.LEADING, mainPanelLayout.createSequentialGroup()
.addGroup(mainPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addGroup(mainPanelLayout.createSequentialGroup()
.addContainerGap()
.addComponent(topLable))
.addGroup(javax.swing.GroupLayout.Alignment.LEADING, mainPanelLayout.createSequentialGroup()
.addGroup(mainPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.TRAILING)
.addComponent(topLable, javax.swing.GroupLayout.Alignment.LEADING)
.addGroup(javax.swing.GroupLayout.Alignment.LEADING, mainPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.TRAILING, false)
.addComponent(servicesPanel, javax.swing.GroupLayout.Alignment.LEADING, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)
.addComponent(freqSlider, javax.swing.GroupLayout.Alignment.LEADING, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE))))
.addGroup(mainPanelLayout.createSequentialGroup()
.addContainerGap()
.addGroup(mainPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.TRAILING, false)
.addComponent(jProgressBar1, javax.swing.GroupLayout.Alignment.LEADING, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)
.addComponent(imageProgressBar, javax.swing.GroupLayout.Alignment.LEADING, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)
.addComponent(servicesPanel, javax.swing.GroupLayout.Alignment.LEADING, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)
.addComponent(freqSlider, javax.swing.GroupLayout.Alignment.LEADING, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE))))
.addGap(173, 173, 173))
.addGroup(mainPanelLayout.createSequentialGroup()
.addContainerGap()
.addComponent(startButton)
.addContainerGap(316, Short.MAX_VALUE))
.addGroup(mainPanelLayout.createSequentialGroup()
.addGap(81, 81, 81)
.addComponent(imageProgressLabel)
.addContainerGap(227, Short.MAX_VALUE))
.addGroup(mainPanelLayout.createSequentialGroup()
.addGap(74, 74, 74)
.addComponent(refreshFreqLabel)
.addContainerGap(219, Short.MAX_VALUE))
.addGroup(mainPanelLayout.createSequentialGroup()
.addGap(84, 84, 84)
.addComponent(fileProgressLabel)
.addContainerGap(238, Short.MAX_VALUE))
.addComponent(startButton))
.addGroup(mainPanelLayout.createSequentialGroup()
.addGap(74, 74, 74)
.addComponent(refreshFreqLabel)))
.addContainerGap(173, Short.MAX_VALUE))
);
mainPanelLayout.setVerticalGroup(
mainPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
@ -236,15 +219,7 @@ public final class IngestTopComponent extends TopComponent implements DataExplor
.addComponent(freqSlider, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
.addComponent(refreshFreqLabel)
.addGap(28, 28, 28)
.addComponent(imageProgressBar, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
.addComponent(imageProgressLabel)
.addGap(27, 27, 27)
.addComponent(jProgressBar1, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
.addComponent(fileProgressLabel)
.addContainerGap(75, Short.MAX_VALUE))
.addContainerGap(198, Short.MAX_VALUE))
);
mainScrollPane.setViewportView(mainPanel);
@ -262,18 +237,39 @@ public final class IngestTopComponent extends TopComponent implements DataExplor
}// </editor-fold>//GEN-END:initComponents
private void startButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_startButtonActionPerformed
if (manager == null)
return;
//pick the services
List<IngestServiceAbstract>servicesToStart = new ArrayList<IngestServiceAbstract>();
List<IngestServiceAbstract> servicesToStart = new ArrayList<IngestServiceAbstract>();
for (IngestServiceAbstract service : services) {
boolean serviceEnabled = serviceStates.get(service.getName());
if (serviceEnabled)
if (serviceEnabled) {
servicesToStart.add(service);
}
}
//pick the image
//TODO which image ?
//for now enqueue all, and manager will skip already enqueued image
//if image has been processed, it will be enqueued again
int[] imageIds = Case.getCurrentCase().getImageIDs();
SleuthkitCase sc = Case.getCurrentCase().getSleuthkitCase();
List<Image> images = new ArrayList<Image>();
for (int imageId : imageIds) {
try {
final Image image = sc.getImageById(imageId);
images.add(image);
} catch (TskException e) {
logger.log(Level.SEVERE, "Error ingesting image, can't retrieve image id: " + Integer.toString(imageId), e);
} catch (SQLException e) {
logger.log(Level.SEVERE, "Error ingesting image, can't retrieve image id: " + Integer.toString(imageId), e);
}
}
//pick the image
//TODO which image ? just enqueue all, and manager will skip already processed image
manager.execute(services, null);
manager.execute(servicesToStart, images);
}//GEN-LAST:event_startButtonActionPerformed
private void freqSliderStateChanged(javax.swing.event.ChangeEvent evt) {//GEN-FIRST:event_freqSliderStateChanged
@ -285,11 +281,7 @@ public final class IngestTopComponent extends TopComponent implements DataExplor
}
}//GEN-LAST:event_freqSliderStateChanged
// Variables declaration - do not modify//GEN-BEGIN:variables
private javax.swing.JLabel fileProgressLabel;
private javax.swing.JSlider freqSlider;
private javax.swing.JProgressBar imageProgressBar;
private javax.swing.JLabel imageProgressLabel;
private javax.swing.JProgressBar jProgressBar1;
private javax.swing.JPanel mainPanel;
private javax.swing.JScrollPane mainScrollPane;
private javax.swing.JLabel refreshFreqLabel;
@ -301,6 +293,7 @@ public final class IngestTopComponent extends TopComponent implements DataExplor
@Override
public void componentOpened() {
logger.log(Level.INFO, "IngestTopComponent opened()");
manager = new IngestManager(this);
}
@Override

1
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/GetAllFilesContentVisitor.java
View File

@ -35,6 +35,7 @@ import org.sleuthkit.datamodel.TskData.FileKnown;
/**
* Visitor for getting all the files to try to index from any Content object.
* Currently gets all non-zero files.
* TODO should be moved to utility module (needs resolve cyclic deps)
*/
class GetAllFilesContentVisitor extends GetFilesContentVisitor {

1
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/GetFilesContentVisitor.java
View File

@ -35,6 +35,7 @@ import org.sleuthkit.datamodel.VolumeSystem;
/**
* Abstract visitor for getting all the files from content
* TODO should be moved to utility module (needs resolve cyclic deps)
*/
public abstract class GetFilesContentVisitor implements ContentVisitor<Collection<FsContent>> {

17
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestService.java
View File

@ -21,6 +21,8 @@ package org.sleuthkit.autopsy.keywordsearch;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.sleuthkit.autopsy.ingest.IngestManager;
import org.sleuthkit.autopsy.ingest.IngestMessage;
import org.sleuthkit.autopsy.ingest.IngestMessage.MessageType;
import org.sleuthkit.autopsy.ingest.IngestServiceFsContent;
import org.sleuthkit.datamodel.FsContent;
@ -29,6 +31,9 @@ public final class KeywordSearchIngestService implements IngestServiceFsContent
private static final Logger logger = Logger.getLogger(KeywordSearchIngestService.class.getName());
private static KeywordSearchIngestService instance = null;
private IngestManager manager;
public static synchronized KeywordSearchIngestService getDefault() {
if (instance == null) {
@ -39,12 +44,19 @@ public final class KeywordSearchIngestService implements IngestServiceFsContent
@Override
public void process(FsContent fsContent) {
logger.log(Level.INFO, "Processing fsContent: " + fsContent.getName());
//logger.log(Level.INFO, "Processing fsContent: " + fsContent.getName());
try {
Thread.sleep(300);
}
catch (InterruptedException e) {}
manager.postMessage(IngestMessage.createMessage(1, MessageType.INFO, this, "Processing " + fsContent.getName()));
}
@Override
public void complete() {
logger.log(Level.INFO, "complete()");
manager.postMessage(IngestMessage.createMessage(1, MessageType.INFO, this, "COMPLETE"));
}
@Override
@ -55,6 +67,9 @@ public final class KeywordSearchIngestService implements IngestServiceFsContent
@Override
public void init(IngestManager manager) {
logger.log(Level.INFO, "init()");
this.manager = manager;
manager.postMessage(IngestMessage.createMessage(1, MessageType.INFO, this, "INIT"));
}
@Override