Added new ewfVerify ingest module

ewfVerify/build.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- You may freely edit this file. See harness/README in the NetBeans platform -->
+<!-- for some information on what you could do (e.g. targets to override). -->
+<!-- If you delete this file and reopen the project it will be recreated. -->
+<project name="org.sleuthkit.autopsy.ewfverify" default="netbeans" basedir=".">
+    <description>Builds, tests, and runs the project org.sleuthkit.autopsy.ewfverify.</description>
+    <import file="nbproject/build-impl.xml"/>
+</project>

ewfVerify/manifest.mf

@@ -0,0 +1,5 @@
+Manifest-Version: 1.0
+OpenIDE-Module: org.sleuthkit.autopsy.ewfverify
+OpenIDE-Module-Localizing-Bundle: org/sleuthkit/autopsy/ewfverify/Bundle.properties
+OpenIDE-Module-Specification-Version: 1.0
+

ewfVerify/nbproject/build-impl.xml

@@ -0,0 +1,45 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+*** GENERATED FROM project.xml - DO NOT EDIT  ***
+***         EDIT ../build.xml INSTEAD         ***
+-->
+<project name="org.sleuthkit.autopsy.ewfverify-impl" basedir="..">
+    <fail message="Please build using Ant 1.7.1 or higher.">
+        <condition>
+            <not>
+                <antversion atleast="1.7.1"/>
+            </not>
+        </condition>
+    </fail>
+    <property file="nbproject/private/suite-private.properties"/>
+    <property file="nbproject/suite.properties"/>
+    <fail unless="suite.dir">You must set 'suite.dir' to point to your containing module suite</fail>
+    <property file="${suite.dir}/nbproject/private/platform-private.properties"/>
+    <property file="${suite.dir}/nbproject/platform.properties"/>
+    <macrodef name="property" uri="http://www.netbeans.org/ns/nb-module-project/2">
+        <attribute name="name"/>
+        <attribute name="value"/>
+        <sequential>
+            <property name="@{name}" value="${@{value}}"/>
+        </sequential>
+    </macrodef>
+    <macrodef name="evalprops" uri="http://www.netbeans.org/ns/nb-module-project/2">
+        <attribute name="property"/>
+        <attribute name="value"/>
+        <sequential>
+            <property name="@{property}" value="@{value}"/>
+        </sequential>
+    </macrodef>
+    <property file="${user.properties.file}"/>
+    <nbmproject2:property name="harness.dir" value="nbplatform.${nbplatform.active}.harness.dir" xmlns:nbmproject2="http://www.netbeans.org/ns/nb-module-project/2"/>
+    <nbmproject2:property name="nbplatform.active.dir" value="nbplatform.${nbplatform.active}.netbeans.dest.dir" xmlns:nbmproject2="http://www.netbeans.org/ns/nb-module-project/2"/>
+    <nbmproject2:evalprops property="cluster.path.evaluated" value="${cluster.path}" xmlns:nbmproject2="http://www.netbeans.org/ns/nb-module-project/2"/>
+    <fail message="Path to 'platform' cluster missing in $${cluster.path} property or using corrupt Netbeans Platform (missing harness).">
+        <condition>
+            <not>
+                <contains string="${cluster.path.evaluated}" substring="platform"/>
+            </not>
+        </condition>
+    </fail>
+    <import file="${harness.dir}/build.xml"/>
+</project>

ewfVerify/nbproject/platform.properties

@@ -0,0 +1,120 @@
+branding.token=autopsy
+netbeans-plat-version=7.3.1
+suite.dir=${basedir}
+nbplatform.active.dir=${suite.dir}/netbeans-plat/${netbeans-plat-version}
+harness.dir=${nbplatform.active.dir}/harness
+bootstrap.url=http://deadlock.netbeans.org/hudson/job/nbms-and-javadoc/lastStableBuild/artifact/nbbuild/netbeans/harness/tasks.jar
+autoupdate.catalog.url=http://dlc.sun.com.edgesuite.net/netbeans/updates/${netbeans-plat-version}/uc/final/distribution/catalog.xml.gz
+cluster.path=\
+    ${nbplatform.active.dir}/harness:\
+    ${nbplatform.active.dir}/java:\
+    ${nbplatform.active.dir}/platform
+disabled.modules=\
+    org.apache.tools.ant.module,\
+    org.netbeans.api.debugger.jpda,\
+    org.netbeans.api.java,\
+    org.netbeans.lib.nbjavac,\
+    org.netbeans.libs.cglib,\
+    org.netbeans.libs.javacapi,\
+    org.netbeans.libs.javacimpl,\
+    org.netbeans.libs.springframework,\
+    org.netbeans.modules.ant.browsetask,\
+    org.netbeans.modules.ant.debugger,\
+    org.netbeans.modules.ant.freeform,\
+    org.netbeans.modules.ant.grammar,\
+    org.netbeans.modules.ant.kit,\
+    org.netbeans.modules.beans,\
+    org.netbeans.modules.classfile,\
+    org.netbeans.modules.dbschema,\
+    org.netbeans.modules.debugger.jpda,\
+    org.netbeans.modules.debugger.jpda.ant,\
+    org.netbeans.modules.debugger.jpda.kit,\
+    org.netbeans.modules.debugger.jpda.projects,\
+    org.netbeans.modules.debugger.jpda.ui,\
+    org.netbeans.modules.debugger.jpda.visual,\
+    org.netbeans.modules.findbugs.installer,\
+    org.netbeans.modules.form,\
+    org.netbeans.modules.form.binding,\
+    org.netbeans.modules.form.j2ee,\
+    org.netbeans.modules.form.kit,\
+    org.netbeans.modules.form.nb,\
+    org.netbeans.modules.form.refactoring,\
+    org.netbeans.modules.hibernate,\
+    org.netbeans.modules.hibernatelib,\
+    org.netbeans.modules.hudson.ant,\
+    org.netbeans.modules.hudson.maven,\
+    org.netbeans.modules.i18n,\
+    org.netbeans.modules.i18n.form,\
+    org.netbeans.modules.j2ee.core.utilities,\
+    org.netbeans.modules.j2ee.eclipselink,\
+    org.netbeans.modules.j2ee.eclipselinkmodelgen,\
+    org.netbeans.modules.j2ee.jpa.refactoring,\
+    org.netbeans.modules.j2ee.jpa.verification,\
+    org.netbeans.modules.j2ee.metadata,\
+    org.netbeans.modules.j2ee.metadata.model.support,\
+    org.netbeans.modules.j2ee.persistence,\
+    org.netbeans.modules.j2ee.persistence.kit,\
+    org.netbeans.modules.j2ee.persistenceapi,\
+    org.netbeans.modules.java.api.common,\
+    org.netbeans.modules.java.debug,\
+    org.netbeans.modules.java.editor,\
+    org.netbeans.modules.java.editor.lib,\
+    org.netbeans.modules.java.examples,\
+    org.netbeans.modules.java.freeform,\
+    org.netbeans.modules.java.guards,\
+    org.netbeans.modules.java.helpset,\
+    org.netbeans.modules.java.hints,\
+    org.netbeans.modules.java.hints.declarative,\
+    org.netbeans.modules.java.hints.declarative.test,\
+    org.netbeans.modules.java.hints.legacy.spi,\
+    org.netbeans.modules.java.hints.test,\
+    org.netbeans.modules.java.hints.ui,\
+    org.netbeans.modules.java.j2seplatform,\
+    org.netbeans.modules.java.j2seproject,\
+    org.netbeans.modules.java.kit,\
+    org.netbeans.modules.java.lexer,\
+    org.netbeans.modules.java.navigation,\
+    org.netbeans.modules.java.platform,\
+    org.netbeans.modules.java.preprocessorbridge,\
+    org.netbeans.modules.java.project,\
+    org.netbeans.modules.java.source,\
+    org.netbeans.modules.java.source.ant,\
+    org.netbeans.modules.java.source.queries,\
+    org.netbeans.modules.java.source.queriesimpl,\
+    org.netbeans.modules.java.sourceui,\
+    org.netbeans.modules.java.testrunner,\
+    org.netbeans.modules.javadoc,\
+    org.netbeans.modules.javawebstart,\
+    org.netbeans.modules.junit,\
+    org.netbeans.modules.maven,\
+    org.netbeans.modules.maven.checkstyle,\
+    org.netbeans.modules.maven.coverage,\
+    org.netbeans.modules.maven.embedder,\
+    org.netbeans.modules.maven.grammar,\
+    org.netbeans.modules.maven.graph,\
+    org.netbeans.modules.maven.hints,\
+    org.netbeans.modules.maven.indexer,\
+    org.netbeans.modules.maven.junit,\
+    org.netbeans.modules.maven.kit,\
+    org.netbeans.modules.maven.model,\
+    org.netbeans.modules.maven.osgi,\
+    org.netbeans.modules.maven.persistence,\
+    org.netbeans.modules.maven.refactoring,\
+    org.netbeans.modules.maven.repository,\
+    org.netbeans.modules.maven.search,\
+    org.netbeans.modules.maven.spring,\
+    org.netbeans.modules.projectimport.eclipse.core,\
+    org.netbeans.modules.projectimport.eclipse.j2se,\
+    org.netbeans.modules.refactoring.java,\
+    org.netbeans.modules.spellchecker.bindings.java,\
+    org.netbeans.modules.spring.beans,\
+    org.netbeans.modules.testng,\
+    org.netbeans.modules.testng.ant,\
+    org.netbeans.modules.testng.maven,\
+    org.netbeans.modules.websvc.jaxws21,\
+    org.netbeans.modules.websvc.jaxws21api,\
+    org.netbeans.modules.websvc.saas.codegen.java,\
+    org.netbeans.modules.xml.jaxb,\
+    org.netbeans.modules.xml.tools.java,\
+    org.netbeans.spi.java.hints
+

ewfVerify/nbproject/project.properties

@@ -0,0 +1,2 @@
+javac.source=1.7
+javac.compilerargs=-Xlint -Xlint:-serial

ewfVerify/nbproject/project.xml

@@ -0,0 +1,31 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://www.netbeans.org/ns/project/1">
+    <type>org.netbeans.modules.apisupport.project</type>
+    <configuration>
+        <data xmlns="http://www.netbeans.org/ns/nb-module-project/3">
+            <code-name-base>org.sleuthkit.autopsy.ewfverify</code-name-base>
+            <suite-component/>
+            <module-dependencies>
+                <dependency>
+                    <code-name-base>org.sleuthkit.autopsy.core</code-name-base>
+                    <build-prerequisite/>
+                    <compile-dependency/>
+                    <run-dependency>
+                        <release-version>9</release-version>
+                        <specification-version>7.0</specification-version>
+                    </run-dependency>
+                </dependency>
+                <dependency>
+                    <code-name-base>org.sleuthkit.autopsy.corelibs</code-name-base>
+                    <build-prerequisite/>
+                    <compile-dependency/>
+                    <run-dependency>
+                        <release-version>3</release-version>
+                        <specification-version>1.1</specification-version>
+                    </run-dependency>
+                </dependency>
+            </module-dependencies>
+            <public-packages/>
+        </data>
+    </configuration>
+</project>

ewfVerify/nbproject/suite.properties

@@ -0,0 +1 @@
+suite.dir=${basedir}/..

ewfVerify/src/org/sleuthkit/autopsy/ewfverify/Bundle.properties

@@ -0,0 +1 @@
+OpenIDE-Module-Name=ewfVerify

ewfVerify/src/org/sleuthkit/autopsy/ewfverify/EwfVerifyIngestModule.java

@@ -0,0 +1,173 @@
+/*
+ * Autopsy Forensic Browser
+ *
+ * Copyright 2013 Basis Technology Corp.
+ * Contact: carrier <at> sleuthkit <dot> org
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.sleuthkit.autopsy.ewfverify;
+
+
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import org.sleuthkit.autopsy.coreutils.StopWatch;
+import org.sleuthkit.autopsy.ingest.IngestDataSourceWorkerController;
+import org.sleuthkit.autopsy.ingest.IngestMessage;
+import org.sleuthkit.autopsy.ingest.IngestMessage.MessageType;
+import org.sleuthkit.autopsy.ingest.IngestModuleDataSource;
+import org.sleuthkit.autopsy.ingest.IngestModuleInit;
+import org.sleuthkit.autopsy.ingest.IngestServices;
+import org.sleuthkit.autopsy.ingest.PipelineContext;
+import org.sleuthkit.datamodel.Content;
+import org.sleuthkit.datamodel.Image;
+import org.sleuthkit.datamodel.TskCoreException;
+import org.sleuthkit.datamodel.TskData;
+
+/**
+ *
+ * @author jwallace
+ */
+public class EwfVerifyIngestModule extends IngestModuleDataSource {
+    private static final String MODULE_NAME = "ewf Verify";
+    private static final String MODULE_VERSION = "1.0";
+    private static final String MODULE_DESCRIPTION = "Validates the integrity of E01 files.";
+    private static final long CHUNK_SIZE = 16 * 1024;
+    private IngestServices services;
+    private volatile boolean running = false;
+    private Image img;
+    private MessageDigest md;
+    private Logger logger;
+    private static int messageId = 0;
+    private volatile boolean cancelled = false;
+    private boolean verified = false;
+
+    public EwfVerifyIngestModule() {
+    }
+    
+    @Override
+    public void process(PipelineContext<IngestModuleDataSource> pipelineContext, Content dataSource, IngestDataSourceWorkerController controller) {
+        try {
+            img = dataSource.getImage();
+        } catch (TskCoreException ex) {
+            img = null;
+            logger.log(Level.SEVERE, "Failed to get image from Content.", ex);
+            services.postMessage(IngestMessage.createMessage(++messageId, MessageType.ERROR, this, "Error processing " + dataSource.getName()));
+        }
+        
+        if (img.getType() != TskData.TSK_IMG_TYPE_ENUM.TSK_IMG_TYPE_EWF_EWF) {
+            img = null;
+            // TODO notify?
+            logger.log(Level.INFO, "Skipping non-ewf image " + img.getName());
+            return;
+        }
+
+        services.postMessage(IngestMessage.createMessage(++messageId, MessageType.INFO, this, "Starting " + dataSource.getName()));
+        long size = img.getSize();      // size of the image
+        
+        // TODO handle size = 0
+        
+        int totalChunks = (int) Math.ceil(size / CHUNK_SIZE);
+        System.out.println("TOTAL CHUNKS = " + totalChunks);
+        int read;
+        
+        // TODO find an appropriate size for this.
+        byte[] data;
+        controller.switchToDeterminate(totalChunks);
+        
+        running = true;
+        StopWatch timer = new StopWatch();
+        timer.start();
+        for (int i = 0; i < totalChunks; i++) {
+            if (cancelled) {
+                timer.stop();
+                running = false;
+                return;
+            }
+            data = new byte[ (int) CHUNK_SIZE ];
+            try {
+                read = img.read(data, i * CHUNK_SIZE, CHUNK_SIZE);
+            } catch (TskCoreException ex) {
+                services.postMessage(IngestMessage.createMessage(++messageId, MessageType.ERROR, this, "Error processing " + img.getName()));
+                logger.log(Level.SEVERE, "Error reading from image: " + img.getName(), ex);
+            }
+            md.update(data);
+            controller.progress(i);
+        }
+        timer.stop();
+        byte[] byteHash = md.digest();
+        String hash = bytesToString(byteHash);
+        System.out.println("MD5 HASH: " + hash);
+        System.out.println("GENERATING HASH TOOK " + timer.getElapsedTimeSecs() + " SECONDS");
+        running = false;
+        // TODO logic to check if it is verified.
+        verified = true;
+    }
+
+    @Override
+    public void init(IngestModuleInit initContext) {
+        services = IngestServices.getDefault();
+        logger = services.getLogger(this);
+        try {
+            md = MessageDigest.getInstance("MD5");
+        } catch (NoSuchAlgorithmException ex) {
+            logger.log(Level.WARNING, "Error getting md5 algorithm", ex);
+            throw new RuntimeException("Failed to get MD5 algorithm");
+        }
+        cancelled = false;
+        running = false;
+        img = null;
+    }
+
+    @Override
+    public void complete() {
+        logger.info("complete() " + this.getName());
+        String msg = verified ? " verified." : " not verified.";
+        services.postMessage(IngestMessage.createMessage(++messageId, MessageType.INFO, this, img.getName() +  msg));
+    }
+
+    @Override
+    public void stop() {
+        cancelled = true;
+    }
+
+    @Override
+    public String getName() {
+        return MODULE_NAME;
+    }
+
+    @Override
+    public String getVersion() {
+        return MODULE_VERSION;
+    }
+
+    @Override
+    public String getDescription() {
+        return MODULE_DESCRIPTION;
+    }
+
+    @Override
+    public boolean hasBackgroundJobsRunning() {
+        return running;
+    }
+
+    private String bytesToString(byte[] byteHash) {
+        StringBuilder sb = new StringBuilder();
+        for (byte b : byteHash) {
+            sb.append(String.format("%02x", b&0xff));
+        }
+        return sb.toString();
+    }
+}

nbproject/project.properties

@@ -1,46 +1,49 @@
-app.icon=branding/core/core.jar/org/netbeans/core/startup/frame48.gif
+app.icon=branding/core/core.jar/org/netbeans/core/startup/frame48.gif
-### Title of the application
+### Title of the application
-app.title=Autopsy
+app.title=Autopsy
-### lowercase version of above
+### lowercase version of above
-app.name=autopsy
+app.name=autopsy
-### if left unset, version will default to today's date
+### if left unset, version will default to today's date
-app.version=3.0.7
+app.version=3.0.7
-### Build type isn't used at this point, but it may be useful
+### Build type isn't used at this point, but it may be useful
-### Must be one of: DEVELOPMENT, RELEASE
+### Must be one of: DEVELOPMENT, RELEASE
-build.type=RELEASE
+build.type=RELEASE
-#build.type=DEVELOPMENT
+project.org.sleuthkit.autopsy.ewfverify=EWFVerify
-update_versions=false
+#build.type=DEVELOPMENT
-#custom JVM options
+update_versions=false
-#Note: can be higher on 64 bit systems, should be in sync with build.xml
+#custom JVM options
-run.args.extra=-J-Xms24m -J-XX:MaxPermSize=128M -J-Xverify:none
+#Note: can be higher on 64 bit systems, should be in sync with build.xml
-auxiliary.org-netbeans-modules-apisupport-installer.license-type=apache.v2
+run.args.extra=-J-Xms24m -J-XX:MaxPermSize=128M -J-Xverify:none
-auxiliary.org-netbeans-modules-apisupport-installer.os-linux=false
+auxiliary.org-netbeans-modules-apisupport-installer.license-type=apache.v2
-auxiliary.org-netbeans-modules-apisupport-installer.os-macosx=false
+auxiliary.org-netbeans-modules-apisupport-installer.os-linux=false
-auxiliary.org-netbeans-modules-apisupport-installer.os-solaris=false
+auxiliary.org-netbeans-modules-apisupport-installer.os-macosx=false
-auxiliary.org-netbeans-modules-apisupport-installer.os-windows=true
+auxiliary.org-netbeans-modules-apisupport-installer.os-solaris=false
-auxiliary.org-netbeans-modules-apisupport-installer.pack200-enabled=false
+auxiliary.org-netbeans-modules-apisupport-installer.os-windows=true
-branding.token=${app.name}
+auxiliary.org-netbeans-modules-apisupport-installer.pack200-enabled=false
-modules=\
+branding.token=${app.name}
-    ${project.org.sleuthkit.autopsy.keywordsearch}:\
+modules=\
-    ${project.org.sleuthkit.autopsy.hashdatabase}:\
+    ${project.org.sleuthkit.autopsy.keywordsearch}:\
-    ${project.org.sleuthkit.autopsy.recentactivity}:\
+    ${project.org.sleuthkit.autopsy.hashdatabase}:\
-    ${project.org.sleuthkit.autopsy.testing}:\
+    ${project.org.sleuthkit.autopsy.recentactivity}:\
-    ${project.org.sleuthkit.autopsy.thunderbirdparser}:\
+    ${project.org.sleuthkit.autopsy.testing}:\
-    ${project.org.sleuthkit.autopsy.exifparser}:\
+    ${project.org.sleuthkit.autopsy.thunderbirdparser}:\
-    ${project.org.sleuthkit.autopsy.core}:\
+    ${project.org.sleuthkit.autopsy.exifparser}:\
-    ${project.org.sleuthkit.autopsy.corelibs}:\
+    ${project.org.sleuthkit.autopsy.core}:\
-    ${project.org.sleuthkit.autopsy.sevenzip}:\
+    ${project.org.sleuthkit.autopsy.corelibs}:\
-    ${project.org.sleuthkit.autopsy.scalpel}:\
+    ${project.org.sleuthkit.autopsy.sevenzip}:\
-    ${project.org.sleuthkit.autopsy.timeline}
+    ${project.org.sleuthkit.autopsy.scalpel}:\
-project.org.sleuthkit.autopsy.core=Core
+    ${project.org.sleuthkit.autopsy.timeline}:\
-project.org.sleuthkit.autopsy.corelibs=CoreLibs
+    ${project.org.sleuthkit.autopsy.ewfverify}
-project.org.sleuthkit.autopsy.hashdatabase=HashDatabase
+project.org.sleuthkit.autopsy.core=Core
-project.org.sleuthkit.autopsy.keywordsearch=KeywordSearch
+project.org.sleuthkit.autopsy.corelibs=CoreLibs
-project.org.sleuthkit.autopsy.recentactivity=RecentActivity
+project.org.sleuthkit.autopsy.hashdatabase=HashDatabase
-project.org.sleuthkit.autopsy.testing=Testing
+project.org.sleuthkit.autopsy.keywordsearch=KeywordSearch
-project.org.sleuthkit.autopsy.thunderbirdparser=thunderbirdparser
+project.org.sleuthkit.autopsy.recentactivity=RecentActivity
-project.org.sleuthkit.autopsy.exifparser=ExifParser
+project.org.sleuthkit.autopsy.testing=Testing
-project.org.sleuthkit.autopsy.sevenzip=SevenZip
+project.org.sleuthkit.autopsy.thunderbirdparser=thunderbirdparser
-project.org.sleuthkit.autopsy.scalpel=ScalpelCarver
+project.org.sleuthkit.autopsy.exifparser=ExifParser
-project.org.sleuthkit.autopsy.timeline=Timeline
+project.org.sleuthkit.autopsy.sevenzip=SevenZip
-
+project.org.sleuthkit.autopsy.scalpel=ScalpelCarver
+project.org.sleuthkit.autopsy.timeline=Timeline
+project.org.sleuthkit.autopsy.ewfverify=ewfVerify
+