Merge develop into auto ingest case deletion branch

Core/src/org/sleuthkit/autopsy/centralrepository/contentviewer/DataContentViewerOtherCases.java

@@ -380,7 +380,7 @@ public class DataContentViewerOtherCases extends JPanel implements DataContentVi
          * a case to be unique. We should improve this in the future.
          */
         Set<String> cases = new HashSet<>();
-        Map<String, String> devices = new HashMap();
+        Map<String, String> devices = new HashMap<>();
         
         for (int i=0; i < model.getRowCount(); i++) {
             String caseName = (String) model.getValueAt(i, caseColumnIndex);

Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java

@@ -62,7 +62,7 @@ abstract class AbstractSqlEamDb implements EamDb {
     static final String SCHEMA_MINOR_VERSION_KEY = "SCHEMA_MINOR_VERSION";
     static final String CREATION_SCHEMA_MAJOR_VERSION_KEY = "CREATION_SCHEMA_MAJOR_VERSION";
     static final String CREATION_SCHEMA_MINOR_VERSION_KEY = "CREATION_SCHEMA_MINOR_VERSION";
-    static final CaseDbSchemaVersionNumber SOFTWARE_CR_DB_SCHEMA_VERSION = new CaseDbSchemaVersionNumber(1, 2);
+    static final CaseDbSchemaVersionNumber SOFTWARE_CR_DB_SCHEMA_VERSION = new CaseDbSchemaVersionNumber(1, 3);
 
     protected final List<CorrelationAttributeInstance.Type> defaultCorrelationTypes;
 
@@ -106,6 +106,11 @@ abstract class AbstractSqlEamDb implements EamDb {
         });
     }
 
+    /**
+     * Setup and create a connection to the selected database implementation
+     */
+    protected abstract Connection connect(boolean foreignKeys) throws EamDbException;
+
     /**
      * Setup and create a connection to the selected database implementation
      */
@@ -3382,12 +3387,13 @@ abstract class AbstractSqlEamDb implements EamDb {
         Statement statement = null;
         PreparedStatement preparedStatement = null;
         Connection conn = null;
+        EamDbPlatformEnum selectedPlatform = null;
         try {
 
-            conn = connect();
+            conn = connect(false);
             conn.setAutoCommit(false);
             statement = conn.createStatement();
-
+            selectedPlatform = EamDbPlatformEnum.getSelectedPlatform();
             int minorVersion = 0;
             String minorVersionStr = null;
             resultSet = statement.executeQuery("SELECT value FROM db_info WHERE name='" + AbstractSqlEamDb.SCHEMA_MINOR_VERSION_KEY + "'");
@@ -3441,8 +3447,6 @@ abstract class AbstractSqlEamDb implements EamDb {
                 return;
             }
 
-            EamDbPlatformEnum selectedPlatform = EamDbPlatformEnum.getSelectedPlatform();
-
             /*
              * Update to 1.1
              */
@@ -3626,7 +3630,36 @@ abstract class AbstractSqlEamDb implements EamDb {
                 statement.execute("INSERT INTO db_info (name, value) VALUES ('" + AbstractSqlEamDb.CREATION_SCHEMA_MAJOR_VERSION_KEY + "','" + creationMajorVer + "')");
                 statement.execute("INSERT INTO db_info (name, value) VALUES ('" + AbstractSqlEamDb.CREATION_SCHEMA_MINOR_VERSION_KEY + "','" + creationMinorVer + "')");
             }
+            /*
+             * Update to 1.3
+             */
+            if (dbSchemaVersion.compareTo(new CaseDbSchemaVersionNumber(1, 3)) < 0) {
+                switch (selectedPlatform) {
+                    case POSTGRESQL:
+                        statement.execute("ALTER TABLE data_sources DROP CONSTRAINT datasource_unique");
+                        //unique constraint for upgraded data_sources table is purposefully different than new data_sources table
+                        statement.execute("ALTER TABLE data_sources ADD CONSTRAINT datasource_unique UNIQUE (case_id, device_id, name, datasource_obj_id)");
 
+                        break;
+                    case SQLITE:
+                        statement.execute("DROP INDEX IF EXISTS data_sources_name");
+                        statement.execute("DROP INDEX IF EXISTS data_sources_object_id");
+                        statement.execute("ALTER TABLE data_sources RENAME TO old_data_sources");
+                        //unique constraint for upgraded data_sources table is purposefully different than new data_sources table
+                        statement.execute("CREATE TABLE IF NOT EXISTS data_sources (id integer primary key autoincrement NOT NULL,"
+                                + "case_id integer NOT NULL,device_id text NOT NULL,name text NOT NULL,datasource_obj_id integer,"
+                                + "md5 text DEFAULT NULL,sha1 text DEFAULT NULL,sha256 text DEFAULT NULL,"
+                                + "foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL,"
+                                + "CONSTRAINT datasource_unique UNIQUE (case_id, device_id, name, datasource_obj_id))");
+                        statement.execute(SqliteEamDbSettings.getAddDataSourcesNameIndexStatement());
+                        statement.execute(SqliteEamDbSettings.getAddDataSourcesObjectIdIndexStatement());
+                        statement.execute("INSERT INTO data_sources SELECT * FROM old_data_sources");
+                        statement.execute("DROP TABLE old_data_sources");
+                        break;
+                    default:
+                        throw new EamDbException("Currently selected database platform \"" + selectedPlatform.name() + "\" can not be upgraded.");
+                }
+            }
             updateSchemaVersion(conn);
             conn.commit();
             logger.log(Level.INFO, String.format("Central Repository schema updated to version %s", SOFTWARE_CR_DB_SCHEMA_VERSION));

Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDb.java

@@ -161,6 +161,21 @@ final class PostgresEamDb extends AbstractSqlEamDb {
         connectionPool.setValidationQuery(dbSettings.getValidationQuery());
     }
 
+    /**
+     * Lazily setup Singleton connection on first request.
+     *
+     * @param foreignKeys -ignored arguement with postgres databases
+     *
+     * @return A connection from the connection pool.
+     *
+     * @throws EamDbException
+     */
+    @Override
+    protected Connection connect(boolean foreignKeys) throws EamDbException {
+        //foreignKeys boolean is ignored for postgres
+        return connect();
+    }
+
     /**
      * Lazily setup Singleton connection on first request.
      *
@@ -179,7 +194,6 @@ final class PostgresEamDb extends AbstractSqlEamDb {
                 setupConnectionPool();
             }
         }
-
         try {
             return connectionPool.getConnection();
         } catch (SQLException ex) {

Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDbSettings.java

@@ -1,7 +1,7 @@
 /*
  * Central Repository
  *
- * Copyright 2015-2017 Basis Technology Corp.
+ * Copyright 2015-2019 Basis Technology Corp.
  * Contact: carrier <at> sleuthkit <dot> org
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
@@ -43,16 +43,16 @@ import static org.sleuthkit.autopsy.centralrepository.datamodel.AbstractSqlEamDb
 public final class PostgresEamDbSettings {
 
     private final static Logger LOGGER = Logger.getLogger(PostgresEamDbSettings.class.getName());
-    private final String DEFAULT_HOST = ""; // NON-NLS
+    private final static String DEFAULT_HOST = ""; // NON-NLS
-    private final int DEFAULT_PORT = 5432;
+    private final static int DEFAULT_PORT = 5432;
-    private final String DEFAULT_DBNAME = "central_repository"; // NON-NLS
+    private final static String DEFAULT_DBNAME = "central_repository"; // NON-NLS
-    private final String DEFAULT_USERNAME = "";
+    private final static String DEFAULT_USERNAME = "";
-    private final String DEFAULT_PASSWORD = "";
+    private final static String DEFAULT_PASSWORD = "";
-    private final String VALIDATION_QUERY = "SELECT version()"; // NON-NLS
+    private final static String VALIDATION_QUERY = "SELECT version()"; // NON-NLS
-    private final String JDBC_BASE_URI = "jdbc:postgresql://"; // NON-NLS
+    private final static String JDBC_BASE_URI = "jdbc:postgresql://"; // NON-NLS
-    private final String JDBC_DRIVER = "org.postgresql.Driver"; // NON-NLS
+    private final static String JDBC_DRIVER = "org.postgresql.Driver"; // NON-NLS
-    private final String DB_NAMES_REGEX = "[a-z][a-z0-9_]*"; // only lower case
+    private final static String DB_NAMES_REGEX = "[a-z][a-z0-9_]*"; // only lower case
-    private final String DB_USER_NAMES_REGEX = "[a-zA-Z]\\w*";
+    private final static String DB_USER_NAMES_REGEX = "[a-zA-Z]\\w*";
     private String host;
     private int port;
     private String dbName;
@@ -339,23 +339,6 @@ public final class PostgresEamDbSettings {
         String casesIdx1 = "CREATE INDEX IF NOT EXISTS cases_org_id ON cases (org_id)";
         String casesIdx2 = "CREATE INDEX IF NOT EXISTS cases_case_uid ON cases (case_uid)";
 
-        StringBuilder createDataSourcesTable = new StringBuilder();
-        createDataSourcesTable.append("CREATE TABLE IF NOT EXISTS data_sources (");
-        createDataSourcesTable.append("id SERIAL PRIMARY KEY,");
-        createDataSourcesTable.append("case_id integer NOT NULL,");
-        createDataSourcesTable.append("device_id text NOT NULL,");
-        createDataSourcesTable.append("name text NOT NULL,");
-        createDataSourcesTable.append("datasource_obj_id BIGINT,");
-        createDataSourcesTable.append("md5 text DEFAULT NULL,");
-        createDataSourcesTable.append("sha1 text DEFAULT NULL,");
-        createDataSourcesTable.append("sha256 text DEFAULT NULL,");
-        createDataSourcesTable.append("foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL,");
-        createDataSourcesTable.append("CONSTRAINT datasource_unique UNIQUE (case_id, device_id, name)");
-        createDataSourcesTable.append(")");
-
-        String dataSourceIdx1 = "CREATE INDEX IF NOT EXISTS data_sources_name ON data_sources (name)";
-        String dataSourceIdx2 = "CREATE INDEX IF NOT EXISTS data_sources_object_id ON data_sources (datasource_obj_id)";
-
         StringBuilder createReferenceSetsTable = new StringBuilder();
         createReferenceSetsTable.append("CREATE TABLE IF NOT EXISTS reference_sets (");
         createReferenceSetsTable.append("id SERIAL PRIMARY KEY,");
@@ -422,10 +405,10 @@ public final class PostgresEamDbSettings {
             stmt.execute(casesIdx1);
             stmt.execute(casesIdx2);
 
-            stmt.execute(createDataSourcesTable.toString());
+            stmt.execute(getCreateDataSourcesTableStatement());
-            stmt.execute(dataSourceIdx1);
+            stmt.execute(getAddDataSourcesNameIndexStatement());
-            stmt.execute(dataSourceIdx2);
+            stmt.execute(getAddDataSourcesObjectIdIndexStatement());
-            
+
             stmt.execute(createReferenceSetsTable.toString());
             stmt.execute(referenceSetsIdx1);
 
@@ -487,21 +470,50 @@ public final class PostgresEamDbSettings {
      */
     static String getCreateArtifactInstancesTableTemplate() {
         // Each "%s" will be replaced with the relevant TYPE_instances table name.
-        StringBuilder createArtifactInstancesTableTemplate = new StringBuilder();
+        return ("CREATE TABLE IF NOT EXISTS %s (id SERIAL PRIMARY KEY,case_id integer NOT NULL,"
-        createArtifactInstancesTableTemplate.append("CREATE TABLE IF NOT EXISTS %s (");
+                + "data_source_id integer NOT NULL,value text NOT NULL,file_path text NOT NULL,"
-        createArtifactInstancesTableTemplate.append("id SERIAL PRIMARY KEY,");
+                + "known_status integer NOT NULL,comment text,file_obj_id BIGINT,"
-        createArtifactInstancesTableTemplate.append("case_id integer NOT NULL,");
+                + "CONSTRAINT %s_multi_unique_ UNIQUE (data_source_id, value, file_path),"
-        createArtifactInstancesTableTemplate.append("data_source_id integer NOT NULL,");
+                + "foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL,"
-        createArtifactInstancesTableTemplate.append("value text NOT NULL,");
+                + "foreign key (data_source_id) references data_sources(id) ON UPDATE SET NULL ON DELETE SET NULL)");
-        createArtifactInstancesTableTemplate.append("file_path text NOT NULL,");
+    }
-        createArtifactInstancesTableTemplate.append("known_status integer NOT NULL,");
+
-        createArtifactInstancesTableTemplate.append("comment text,");
+    /**
-        createArtifactInstancesTableTemplate.append("file_obj_id BIGINT,");
+     * Get the statement String for creating a new data_sources table in a
-        createArtifactInstancesTableTemplate.append("CONSTRAINT %s_multi_unique_ UNIQUE (data_source_id, value, file_path),");
+     * Postgres central repository.
-        createArtifactInstancesTableTemplate.append("foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL,");
+     *
-        createArtifactInstancesTableTemplate.append("foreign key (data_source_id) references data_sources(id) ON UPDATE SET NULL ON DELETE SET NULL");
+     * @return a String which is a statement for cretating a new data_sources
-        createArtifactInstancesTableTemplate.append(")");
+     *         table
-        return createArtifactInstancesTableTemplate.toString();
+     */
+    static String getCreateDataSourcesTableStatement() {
+        return "CREATE TABLE IF NOT EXISTS data_sources "
+                + "(id SERIAL PRIMARY KEY,case_id integer NOT NULL,device_id text NOT NULL,"
+                + "name text NOT NULL,datasource_obj_id BIGINT,md5 text DEFAULT NULL,"
+                + "sha1 text DEFAULT NULL,sha256 text DEFAULT NULL,"
+                + "foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL,"
+                + "CONSTRAINT datasource_unique UNIQUE (case_id, datasource_obj_id))";
+    }
+
+    /**
+     * Get the statement for creating an index on the name column of the
+     * data_sources table.
+     *
+     * @return a String which is a statement for adding an index on the name
+     *         column of the data_sources table.
+     */
+    static String getAddDataSourcesNameIndexStatement() {
+        return "CREATE INDEX IF NOT EXISTS data_sources_name ON data_sources (name)";
+    }
+
+    /**
+     * Get the statement for creating an index on the data_sources_object_id
+     * column of the data_sources table.
+     *
+     * @return a String which is a statement for adding an index on the
+     *         data_sources_object_id column of the data_sources table.
+     */
+    static String getAddDataSourcesObjectIdIndexStatement() {
+        return "CREATE INDEX IF NOT EXISTS data_sources_object_id ON data_sources (datasource_obj_id)";
     }
 
     /**
@@ -561,8 +573,8 @@ public final class PostgresEamDbSettings {
      * instance table. %s will exist in the template where the name of the new
      * table will be addedd.
      *
-     * @return a String which is a template for adding an index to the file_obj_id
+     * @return a String which is a template for adding an index to the
-     *         column of a _instances table
+     *         file_obj_id column of a _instances table
      */
     static String getAddObjectIdIndexTemplate() {
         // Each "%s" will be replaced with the relevant TYPE_instances table name.

Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDb.java

@@ -153,7 +153,7 @@ final class SqliteEamDb extends AbstractSqlEamDb {
      * Setup a connection pool for db connections.
      *
      */
-    private void setupConnectionPool() throws EamDbException {
+    private void setupConnectionPool(boolean foreignKeysEnabled) throws EamDbException {
 
         if (dbSettings.dbFileExists() == false) {
             throw new EamDbException("Central repository database missing");
@@ -169,27 +169,31 @@ final class SqliteEamDb extends AbstractSqlEamDb {
         connectionPool.setMaxIdle(-1);
         connectionPool.setMaxWaitMillis(1000);
         connectionPool.setValidationQuery(dbSettings.getValidationQuery());
-        connectionPool.setConnectionInitSqls(Arrays.asList("PRAGMA foreign_keys = ON"));
+        if (foreignKeysEnabled) {
+            connectionPool.setConnectionInitSqls(Arrays.asList("PRAGMA foreign_keys = ON"));
+        } else {
+            connectionPool.setConnectionInitSqls(Arrays.asList("PRAGMA foreign_keys = OFF"));
+        }
     }
 
     /**
      * Lazily setup Singleton connection on first request.
      *
+     * @param foreignKeys determines if foreign keys should be enforced during this connection for SQLite
+     *
      * @return A connection from the connection pool.
      *
      * @throws EamDbException
      */
     @Override
-    protected Connection connect() throws EamDbException {
+    protected Connection connect(boolean foreignKeys) throws EamDbException {
         synchronized (this) {
             if (!EamDb.isEnabled()) {
                 throw new EamDbException("Central Repository module is not enabled"); // NON-NLS
             }
-
             if (connectionPool == null) {
-                setupConnectionPool();
+                setupConnectionPool(foreignKeys);
             }
-
             try {
                 return connectionPool.getConnection();
             } catch (SQLException ex) {
@@ -198,6 +202,18 @@ final class SqliteEamDb extends AbstractSqlEamDb {
         }
     }
 
+    /**
+     * Lazily setup Singleton connection on first request with foreign keys enforced.
+     *
+     * @return A connection from the connection pool.
+     *
+     * @throws EamDbException
+     */
+    @Override
+    protected Connection connect() throws EamDbException {
+        return connect(true);
+    }
+    
     @Override
     protected String getConflictClause() {
         // For sqlite, our conflict clause is part of the table schema

Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDbSettings.java

@@ -1,7 +1,7 @@
 /*
  * Central Repository
  *
- * Copyright 2015-2017 Basis Technology Corp.
+ * Copyright 2015-2019 Basis Technology Corp.
  * Contact: carrier <at> sleuthkit <dot> org
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
@@ -43,19 +43,19 @@ import static org.sleuthkit.autopsy.centralrepository.datamodel.AbstractSqlEamDb
 public final class SqliteEamDbSettings {
 
     private final static Logger LOGGER = Logger.getLogger(SqliteEamDbSettings.class.getName());
-    private final String DEFAULT_DBNAME = "central_repository.db"; // NON-NLS
+    private final static String DEFAULT_DBNAME = "central_repository.db"; // NON-NLS
-    private final String DEFAULT_DBDIRECTORY = PlatformUtil.getUserDirectory() + File.separator + "central_repository"; // NON-NLS
+    private final static String DEFAULT_DBDIRECTORY = PlatformUtil.getUserDirectory() + File.separator + "central_repository"; // NON-NLS
-    private final String JDBC_DRIVER = "org.sqlite.JDBC"; // NON-NLS
+    private final static String JDBC_DRIVER = "org.sqlite.JDBC"; // NON-NLS
-    private final String JDBC_BASE_URI = "jdbc:sqlite:"; // NON-NLS
+    private final static String JDBC_BASE_URI = "jdbc:sqlite:"; // NON-NLS
-    private final String VALIDATION_QUERY = "SELECT count(*) from sqlite_master"; // NON-NLS
+    private final static String VALIDATION_QUERY = "SELECT count(*) from sqlite_master"; // NON-NLS
-    private static final String PRAGMA_SYNC_OFF = "PRAGMA synchronous = OFF";
+    private final static String PRAGMA_SYNC_OFF = "PRAGMA synchronous = OFF";
-    private static final String PRAGMA_SYNC_NORMAL = "PRAGMA synchronous = NORMAL";
+    private final static String PRAGMA_SYNC_NORMAL = "PRAGMA synchronous = NORMAL";
-    private static final String PRAGMA_JOURNAL_WAL = "PRAGMA journal_mode = WAL";
+    private final static String PRAGMA_JOURNAL_WAL = "PRAGMA journal_mode = WAL";
-    private static final String PRAGMA_READ_UNCOMMITTED_TRUE = "PRAGMA read_uncommitted = True";
+    private final static String PRAGMA_READ_UNCOMMITTED_TRUE = "PRAGMA read_uncommitted = True";
-    private static final String PRAGMA_ENCODING_UTF8 = "PRAGMA encoding = 'UTF-8'";
+    private final static String PRAGMA_ENCODING_UTF8 = "PRAGMA encoding = 'UTF-8'";
-    private static final String PRAGMA_PAGE_SIZE_4096 = "PRAGMA page_size = 4096";
+    private final static String PRAGMA_PAGE_SIZE_4096 = "PRAGMA page_size = 4096";
-    private static final String PRAGMA_FOREIGN_KEYS_ON = "PRAGMA foreign_keys = ON";
+    private final static String PRAGMA_FOREIGN_KEYS_ON = "PRAGMA foreign_keys = ON";
-    private final String DB_NAMES_REGEX = "[a-z][a-z0-9_]*(\\.db)?";
+    private final static String DB_NAMES_REGEX = "[a-z][a-z0-9_]*(\\.db)?";
     private String dbName;
     private String dbDirectory;
     private int bulkThreshold;
@@ -282,23 +282,6 @@ public final class SqliteEamDbSettings {
         String casesIdx1 = "CREATE INDEX IF NOT EXISTS cases_org_id ON cases (org_id)";
         String casesIdx2 = "CREATE INDEX IF NOT EXISTS cases_case_uid ON cases (case_uid)";
 
-        StringBuilder createDataSourcesTable = new StringBuilder();
-        createDataSourcesTable.append("CREATE TABLE IF NOT EXISTS data_sources (");
-        createDataSourcesTable.append("id integer primary key autoincrement NOT NULL,");
-        createDataSourcesTable.append("case_id integer NOT NULL,");
-        createDataSourcesTable.append("device_id text NOT NULL,");
-        createDataSourcesTable.append("name text NOT NULL,");
-        createDataSourcesTable.append("datasource_obj_id integer,");
-        createDataSourcesTable.append("md5 text DEFAULT NULL,");
-        createDataSourcesTable.append("sha1 text DEFAULT NULL,");
-        createDataSourcesTable.append("sha256 text DEFAULT NULL,");
-        createDataSourcesTable.append("foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL,");
-        createDataSourcesTable.append("CONSTRAINT datasource_unique UNIQUE (case_id, device_id, name)");
-        createDataSourcesTable.append(")");
-
-        String dataSourceIdx1 = "CREATE INDEX IF NOT EXISTS data_sources_name ON data_sources (name)";
-        String dataSourceIdx2 = "CREATE INDEX IF NOT EXISTS data_sources_object_id ON data_sources (datasource_obj_id)";
-
         StringBuilder createReferenceSetsTable = new StringBuilder();
         createReferenceSetsTable.append("CREATE TABLE IF NOT EXISTS reference_sets (");
         createReferenceSetsTable.append("id integer primary key autoincrement NOT NULL,");
@@ -371,9 +354,9 @@ public final class SqliteEamDbSettings {
             stmt.execute(casesIdx1);
             stmt.execute(casesIdx2);
 
-            stmt.execute(createDataSourcesTable.toString());
+            stmt.execute(getCreateDataSourcesTableStatement());
-            stmt.execute(dataSourceIdx1);
+            stmt.execute(getAddDataSourcesNameIndexStatement());
-            stmt.execute(dataSourceIdx2);
+            stmt.execute(getAddDataSourcesObjectIdIndexStatement());
 
             stmt.execute(createReferenceSetsTable.toString());
             stmt.execute(referenceSetsIdx1);
@@ -435,21 +418,49 @@ public final class SqliteEamDbSettings {
      */
     static String getCreateArtifactInstancesTableTemplate() {
         // Each "%s" will be replaced with the relevant TYPE_instances table name.
-        StringBuilder createArtifactInstancesTableTemplate = new StringBuilder();
+        return "CREATE TABLE IF NOT EXISTS %s (id integer primary key autoincrement NOT NULL,"
-        createArtifactInstancesTableTemplate.append("CREATE TABLE IF NOT EXISTS %s (");
+                + "case_id integer NOT NULL,data_source_id integer NOT NULL,value text NOT NULL,"
-        createArtifactInstancesTableTemplate.append("id integer primary key autoincrement NOT NULL,");
+                + "file_path text NOT NULL,known_status integer NOT NULL,comment text,file_obj_id integer,"
-        createArtifactInstancesTableTemplate.append("case_id integer NOT NULL,");
+                + "CONSTRAINT %s_multi_unique UNIQUE(data_source_id, value, file_path) ON CONFLICT IGNORE,"
-        createArtifactInstancesTableTemplate.append("data_source_id integer NOT NULL,");
+                + "foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL,"
-        createArtifactInstancesTableTemplate.append("value text NOT NULL,");
+                + "foreign key (data_source_id) references data_sources(id) ON UPDATE SET NULL ON DELETE SET NULL)";
-        createArtifactInstancesTableTemplate.append("file_path text NOT NULL,");
+    }
-        createArtifactInstancesTableTemplate.append("known_status integer NOT NULL,");
+
-        createArtifactInstancesTableTemplate.append("comment text,");
+    /**
-        createArtifactInstancesTableTemplate.append("file_obj_id integer,");
+     * Get the statement String for creating a new data_sources table in a
-        createArtifactInstancesTableTemplate.append("CONSTRAINT %s_multi_unique UNIQUE(data_source_id, value, file_path) ON CONFLICT IGNORE,");
+     * Sqlite central repository.
-        createArtifactInstancesTableTemplate.append("foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL,");
+     *
-        createArtifactInstancesTableTemplate.append("foreign key (data_source_id) references data_sources(id) ON UPDATE SET NULL ON DELETE SET NULL");
+     * @return a String which is a statement for cretating a new data_sources
-        createArtifactInstancesTableTemplate.append(")");
+     *         table
-        return createArtifactInstancesTableTemplate.toString();
+     */
+    static String getCreateDataSourcesTableStatement() {
+        return "CREATE TABLE IF NOT EXISTS data_sources (id integer primary key autoincrement NOT NULL,"
+                + "case_id integer NOT NULL,device_id text NOT NULL,name text NOT NULL,datasource_obj_id integer,"
+                + "md5 text DEFAULT NULL,sha1 text DEFAULT NULL,sha256 text DEFAULT NULL,"
+                + "foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL,"
+                + "CONSTRAINT datasource_unique UNIQUE (case_id, datasource_obj_id))";
+    }
+
+    /**
+     * Get the statement for creating an index on the name column of the
+     * data_sources table.
+     *
+     * @return a String which is a statement for adding an index on the name
+     *         column of the data_sources table.
+     */
+    static String getAddDataSourcesNameIndexStatement() {
+        return "CREATE INDEX IF NOT EXISTS data_sources_name ON data_sources (name)";
+    }
+
+    /**
+     * Get the statement for creating an index on the data_sources_object_id
+     * column of the data_sources table.
+     *
+     * @return a String which is a statement for adding an index on the
+     *         data_sources_object_id column of the data_sources table.
+     */
+    static String getAddDataSourcesObjectIdIndexStatement() {
+        return "CREATE INDEX IF NOT EXISTS data_sources_object_id ON data_sources (datasource_obj_id)";
     }
 
     /**

KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties

@@ -1,5 +1,5 @@
 OpenIDE-Module-Display-Category=Ingest Module
-OpenIDE-Module-Long-Description=Keyword Search ingest module.\n\n\The module indexes files found in the disk image at ingest time.\n\It then periodically runs the search on the indexed files using one or more keyword lists (containing pure words and/or regular expressions) and posts results.\n\n\The module also contains additional tools integrated in the main GUI, such as keyword list configuration, keyword seach bar in the top-right corner, extracted text viewer and search results viewer showing highlighted keywords found.
+OpenIDE-Module-Long-Description=Keyword Search ingest module.\n\nThe module indexes files found in the disk image at ingest time.\nIt then periodically runs the search on the indexed files using one or more keyword lists (containing pure words and/or regular expressions) and posts results.\n\n\The module also contains additional tools integrated in the main GUI, such as keyword list configuration, keyword search bar in the top-right corner, extracted text viewer and search results viewer showing highlighted keywords found.
 OpenIDE-Module-Name=KeywordSearch
 OptionsCategory_Name_KeywordSearchOptions=Keyword Search
 OptionsCategory_Keywords_KeywordSearchOptions=Keyword Search

KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED

@@ -34,7 +34,7 @@ KeywordSearchIngestModule.startupMessage.failedToGetIndexSchema=Failed to get sc
 KeywordSearchResultFactory.createNodeForKey.noResultsFound.text=No results found.
 KeywordSearchResultFactory.query.exception.msg=Could not perform the query 
 OpenIDE-Module-Display-Category=Ingest Module
-OpenIDE-Module-Long-Description=Keyword Search ingest module.\n\nThe module indexes files found in the disk image at ingest time.\nIt then periodically runs the search on the indexed files using one or more keyword lists (containing pure words and/or regular expressions) and posts results.\n\nThe module also contains additional tools integrated in the main GUI, such as keyword list configuration, keyword seach bar in the top-right corner, extracted text viewer and search results viewer showing highlighted keywords found.
+OpenIDE-Module-Long-Description=Keyword Search ingest module.\n\nThe module indexes files found in the disk image at ingest time.\nIt then periodically runs the search on the indexed files using one or more keyword lists (containing pure words and/or regular expressions) and posts results.\n\n\The module also contains additional tools integrated in the main GUI, such as keyword list configuration, keyword search bar in the top-right corner, extracted text viewer and search results viewer showing highlighted keywords found.
 OpenIDE-Module-Name=KeywordSearch
 OptionsCategory_Name_KeywordSearchOptions=Keyword Search
 OptionsCategory_Keywords_KeywordSearchOptions=Keyword Search

RecentActivity/ivy.xml

@@ -6,7 +6,4 @@
         <conf name="recent-activity"/>
       
     </configurations>
-	<dependencies>
-		<dependency conf="recent-activity->default" org="com.googlecode.plist" name="dd-plist" rev="1.20"/>
-	</dependencies>
 </ivy-module>

RecentActivity/nbproject/project.properties

@@ -1,4 +1,3 @@
-file.reference.dd-plist-1.20.jar=release/modules/ext/dd-plist-1.20.jar
 javac.source=1.8
 javac.compilerargs=-Xlint -Xlint:-serial
 license.file=../LICENSE-2.0.txt

RecentActivity/nbproject/project.xml

@@ -74,10 +74,6 @@
                 </dependency>
             </module-dependencies>
             <public-packages/>
-            <class-path-extension>
-                <runtime-relative-path>ext/dd-plist-1.20.jar</runtime-relative-path>
-                <binary-origin>release/modules/ext/dd-plist-1.20.jar</binary-origin>
-            </class-path-extension>
         </data>
     </configuration>
 </project>

RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Bundle.properties-MERGED

@@ -43,6 +43,7 @@ ExtractOs.unitedLinuxVolume.label=OS Drive (Linux United Linux)
 ExtractOs.windowsVolume.label=OS Drive (Windows)
 ExtractOs.yellowDogLinuxOs.label=Linux (Yellow Dog)
 ExtractOs.yellowDogLinuxVolume.label=OS Drive (Linux Yellow Dog)
+ExtractOS_progressMessage=Checking for OS
 ExtractSafari_Error_Getting_History=An error occurred while processing Safari history files.
 ExtractSafari_Error_Parsing_Bookmark=An error occured while processing Safari Bookmark files
 ExtractSafari_Error_Parsing_Cookies=An error occured while processing Safari Cookies files
@@ -119,6 +120,37 @@ Firefox.getDlPre24.errMsg.errParsingArtifacts={0}: Error parsing {1} Firefox web
 Firefox.getDlV24.errMsg.errFetchFiles=Error fetching 'downloads' files for Firefox.
 Firefox.getDlV24.errMsg.errAnalyzeFile={0}: Error while trying to analyze file:{1}
 Firefox.getDlV24.errMsg.errParsingArtifacts={0}: Error parsing {1} Firefox web download artifacts.
+Progress_Message_Analyze_Registry=Analyzing Registry Files
+Progress_Message_Analyze_Usage=Data Sources Usage Analysis
+Progress_Message_Chrome_AutoFill=Chrome Auto Fill
+Progress_Message_Chrome_Bookmarks=Chrome Bookmarks
+Progress_Message_Chrome_Cookies=Chrome Cookies
+Progress_Message_Chrome_Downloads=Chrome Downloads
+Progress_Message_Chrome_FormHistory=Chrome Form History
+Progress_Message_Chrome_History=Chrome History
+Progress_Message_Chrome_Logins=Chrome Logins
+Progress_Message_Edge_Bookmarks=Microsoft Edge Bookmarks
+Progress_Message_Edge_Cookies=Microsoft Edge Cookies
+Progress_Message_Edge_History=Microsoft Edge History
+Progress_Message_Extract_Resent_Docs=Recent Documents
+Progress_Message_Find_Search_Query=Find Search Queries
+Progress_Message_Firefox_AutoFill=Firefox Auto Fill
+Progress_Message_Firefox_Bookmarks=Firefox Bookmarks
+Progress_Message_Firefox_Cookies=Firefox Cookies
+Progress_Message_Firefox_Downloads=Firefox Downloads
+Progress_Message_Firefox_FormHistory=Firefox Form History
+Progress_Message_Firefox_History=Firefox History
+Progress_Message_IE_AutoFill=IE Auto Fill
+Progress_Message_IE_Bookmarks=IE Bookmarks
+Progress_Message_IE_Cookies=IE Cookies
+Progress_Message_IE_Downloads=IE Downloads
+Progress_Message_IE_FormHistory=IE Form History
+Progress_Message_IE_History=IE History
+Progress_Message_IE_Logins=IE Logins
+Progress_Message_Safari_Bookmarks=Safari Bookmarks
+Progress_Message_Safari_Cookies=Safari Cookies
+Progress_Message_Safari_Downloads=Safari Downloads
+Progress_Message_Safari_History=Safari History
 RAImageIngestModule.process.started=Started {0}
 RAImageIngestModule.process.errModFailed={0} failed - see log for details <br>
 RAImageIngestModule.process.errModErrs={0} had errors -- see log

RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Chrome.java

@@ -37,11 +37,13 @@ import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.FileReader;
 import java.io.IOException;
+import org.openide.util.NbBundle.Messages;
 import org.sleuthkit.autopsy.casemodule.Case;
 import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException;
 import org.sleuthkit.autopsy.casemodule.services.FileManager;
 import org.sleuthkit.autopsy.coreutils.Logger;
 import org.sleuthkit.autopsy.coreutils.NetworkUtils;
+import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProgress;
 import org.sleuthkit.autopsy.ingest.IngestJobContext;
 import org.sleuthkit.autopsy.ingest.ModuleDataEvent;
 import org.sleuthkit.datamodel.AbstractFile;
@@ -81,21 +83,43 @@ class Chrome extends Extract {
     private final Logger logger = Logger.getLogger(this.getClass().getName());
     private Content dataSource;
     private IngestJobContext context;
+    
+    @Messages({
+        "Progress_Message_Chrome_History=Chrome History",
+        "Progress_Message_Chrome_Bookmarks=Chrome Bookmarks",
+        "Progress_Message_Chrome_Cookies=Chrome Cookies",
+        "Progress_Message_Chrome_Downloads=Chrome Downloads",
+        "Progress_Message_Chrome_FormHistory=Chrome Form History",
+        "Progress_Message_Chrome_AutoFill=Chrome Auto Fill",
+        "Progress_Message_Chrome_Logins=Chrome Logins",
+    })
 
     Chrome() {
         moduleName = NbBundle.getMessage(Chrome.class, "Chrome.moduleName");
     }
 
     @Override
-    public void process(Content dataSource, IngestJobContext context) {
+    public void process(Content dataSource, IngestJobContext context, DataSourceIngestModuleProgress progressBar) {
         this.dataSource = dataSource;
         this.context = context;
         dataFound = false;
+        
+        progressBar.progress(Bundle.Progress_Message_Chrome_History());
         this.getHistory();
+        
+        progressBar.progress(Bundle.Progress_Message_Chrome_Bookmarks());
         this.getBookmark();
+        
+        progressBar.progress(Bundle.Progress_Message_Chrome_Cookies());
         this.getCookie();
+        
+        progressBar.progress(Bundle.Progress_Message_Chrome_Logins());
         this.getLogins();
+        
+        progressBar.progress(Bundle.Progress_Message_Chrome_AutoFill());
         this.getAutofill();
+        
+        progressBar.progress(Bundle.Progress_Message_Chrome_Downloads());
         this.getDownload();
         
         ChromeCacheExtractor chromeCacheExtractor = new ChromeCacheExtractor(dataSource, context);

RecentActivity/src/org/sleuthkit/autopsy/recentactivity/DataSourceUsageAnalyzer.java

@@ -26,6 +26,7 @@ import org.apache.commons.io.FilenameUtils;
 import org.openide.util.NbBundle.Messages;
 import org.sleuthkit.autopsy.casemodule.services.FileManager;
 import org.sleuthkit.autopsy.coreutils.Logger;
+import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProgress;
 import org.sleuthkit.autopsy.ingest.IngestJobContext;
 import org.sleuthkit.datamodel.AbstractFile;
 import org.sleuthkit.datamodel.BlackboardArtifact;
@@ -46,12 +47,14 @@ class DataSourceUsageAnalyzer extends Extract {
 
     @Messages({
         "# {0} - OS name",
-        "DataSourceUsageAnalyzer.customVolume.label=OS Drive ({0})"
+        "DataSourceUsageAnalyzer.customVolume.label=OS Drive ({0})",
+        "Progress_Message_Analyze_Usage=Data Sources Usage Analysis",
     })
     @Override
-    void process(Content dataSource, IngestJobContext context) {
+    void process(Content dataSource, IngestJobContext context, DataSourceIngestModuleProgress progressBar) {
         this.dataSource = dataSource;
         try {
+            progressBar.progress(Bundle.Progress_Message_Analyze_Usage());
             createDataSourceUsageArtifacts();
         } catch (TskCoreException ex) {
             logger.log(Level.WARNING, "Failed to check if datasource contained a volume with operating system specific files", ex);

RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Extract.java

@@ -44,6 +44,7 @@ import org.sleuthkit.autopsy.coreutils.Logger;
 import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil;
 import org.sleuthkit.autopsy.coreutils.SQLiteDBConnect;
 import org.sleuthkit.autopsy.datamodel.ContentUtils;
+import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProgress;
 import org.sleuthkit.autopsy.ingest.IngestJobContext;
 import org.sleuthkit.autopsy.ingest.IngestModule.IngestModuleException;
 import org.sleuthkit.datamodel.AbstractFile;
@@ -85,7 +86,7 @@ abstract class Extract {
     void configExtractor() throws IngestModuleException  {        
     }
 
-    abstract void process(Content dataSource, IngestJobContext context);
+    abstract void process(Content dataSource, IngestJobContext context, DataSourceIngestModuleProgress progressBar);
 
     void complete() {
     }

RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractEdge.java

@@ -38,11 +38,13 @@ import org.openide.util.NbBundle.Messages;
 import org.sleuthkit.autopsy.casemodule.Case;
 import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException;
 import org.sleuthkit.autopsy.coreutils.ExecUtil;
+import org.sleuthkit.autopsy.coreutils.FileUtil;
 import org.sleuthkit.autopsy.coreutils.Logger;
 import org.sleuthkit.autopsy.coreutils.NetworkUtils;
 import org.sleuthkit.autopsy.coreutils.PlatformUtil;
 import org.sleuthkit.autopsy.datamodel.ContentUtils;
 import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProcessTerminator;
+import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProgress;
 import org.sleuthkit.autopsy.ingest.IngestJobContext;
 import org.sleuthkit.autopsy.ingest.IngestServices;
 import org.sleuthkit.autopsy.ingest.ModuleDataEvent;
@@ -94,11 +96,13 @@ final class ExtractEdge extends Extract {
     private static final String EDGE_FAVORITE_FILE_NAME = "Favorites.csv"; //NON-NLS
     private static final String EDGE_OUTPUT_FILE_NAME = "Output.txt"; //NON-NLS
     private static final String EDGE_ERROR_FILE_NAME = "File.txt"; //NON-NLS
+    private static final String EDGE_WEBCACHE_FOLDER_NAME = "WebCache"; //NON-NLS
+    private static final String EDGE_SPARTAN_FOLDER_NAME = "MicrosoftEdge"; //NON-NLS
 
     private static final String ESE_TOOL_FOLDER = "ESEDatabaseView"; //NON-NLS
     private static final String EDGE_RESULT_FOLDER_NAME = "results"; //NON-NLS
 
-    private static final SimpleDateFormat DATE_FORMATTER = new SimpleDateFormat("MM/dd/yyyy hh:mm:ss a");
+    private static final SimpleDateFormat DATE_FORMATTER = new SimpleDateFormat("MM/dd/yyyy hh:mm:ss a"); //NON-NLS
 
     @Messages({
         "ExtractEdge_process_errMsg_unableFindESEViewer=Unable to find ESEDatabaseViewer",
@@ -107,6 +111,9 @@ final class ExtractEdge extends Extract {
         "ExtractEdge_process_errMsg_spartanFail=Failure processing Microsoft Edge spartan.edb file",
         "ExtractEdge_Module_Name=Microsoft Edge",
         "ExtractEdge_getHistory_containerFileNotFound=Error while trying to analyze Edge history",
+        "Progress_Message_Edge_History=Microsoft Edge History",
+        "Progress_Message_Edge_Bookmarks=Microsoft Edge Bookmarks",
+        "Progress_Message_Edge_Cookies=Microsoft Edge Cookies",
     })
 
     /**
@@ -122,7 +129,7 @@ final class ExtractEdge extends Extract {
     }
 
     @Override
-    void process(Content dataSource, IngestJobContext context) {
+    void process(Content dataSource, IngestJobContext context, DataSourceIngestModuleProgress progressBar) {
         this.dataSource = dataSource;
         this.context = context;
         this.setFoundData(false);
@@ -164,30 +171,31 @@ final class ExtractEdge extends Extract {
         }
 
         try {
-            this.processWebCacheDbFile(esedumper, webCacheFiles);
+            this.processWebCacheDbFile(esedumper, webCacheFiles, progressBar);
         } catch (IOException | TskCoreException ex) {
             this.addErrorMessage(Bundle.ExtractEdge_process_errMsg_webcacheFail());
             LOG.log(Level.SEVERE, "Error returned from processWebCacheDbFile", ex); // NON-NLS
         }
 
+        progressBar.progress(Bundle.Progress_Message_Edge_Bookmarks());
         try {
             this.processSpartanDbFile(esedumper, spartanFiles);
         } catch (IOException | TskCoreException ex) {
-            this.addErrorMessage(Bundle.ExtractEdge_process_errMsg_webcacheFail());
+            this.addErrorMessage(Bundle.ExtractEdge_process_errMsg_spartanFail());
             LOG.log(Level.SEVERE, "Error returned from processSpartanDbFile", ex); // NON-NLS
         }
     }
 
     /**
-     * Dump the tables from WebCacheV01.dat and look for the data contained with
+     * Process WebCacheV01.dat ese database file creating artifacts for cookies,
-     * in those files including downloads, cookies and history.
+     * and history contained within.
      *
      * @param eseDumperPath Path to ESEDatabaseView.exe
      * @param webCacheFiles List of case WebCacheV01.dat files
      * @throws IOException
      * @throws TskCoreException
      */
-    void processWebCacheDbFile(String eseDumperPath, List<AbstractFile> webCacheFiles) throws IOException, TskCoreException {
+    void processWebCacheDbFile(String eseDumperPath, List<AbstractFile> webCacheFiles, DataSourceIngestModuleProgress progressBar) throws IOException, TskCoreException {
 
         for (AbstractFile webCacheFile : webCacheFiles) {
 
@@ -218,30 +226,29 @@ final class ExtractEdge extends Extract {
                 if (context.dataSourceIngestIsCancelled()) {
                     return;
                 }
-
+                
+                progressBar.progress(Bundle.Progress_Message_Edge_History());
+                
                 this.getHistory(webCacheFile, resultsDir);
 
                 if (context.dataSourceIngestIsCancelled()) {
                     return;
                 }
 
+                progressBar.progress(Bundle.Progress_Message_Edge_Cookies());
+                
                 this.getCookies(webCacheFile, resultsDir);
 
-//                if (context.dataSourceIngestIsCancelled()) {
-//                    return;
-//                }
-//                Putting downloads on hold
-//                this.getDownload(webCacheFile, resultsDir); 
             } finally {
                 tempWebCacheFile.delete();
-                resultsDir.delete();
+                FileUtil.deleteFileDir(resultsDir);
             }
         }
     }
 
     /**
-     * Creates a temp version of the database and runs the ESEDatabaseView tool
+     * Process spartan.edb ese database file creating artifacts for the bookmarks
-     * to dump each of the database tables into a temporary folder.
+     * contained within.
      *
      * @param eseDumperPath Path to ESEDatabaseViewer
      * @param spartanFiles List of the case spartan.edb files
@@ -282,7 +289,7 @@ final class ExtractEdge extends Extract {
 
             } finally {
                 tempSpartanFile.delete();
-                resultsDir.delete();
+                FileUtil.deleteFileDir(resultsDir);
             }
         }
     }
@@ -392,7 +399,7 @@ final class ExtractEdge extends Extract {
     }
 
     /**
-     * Queries for cookie files and adds artifacts
+     * Queries for cookie files and adds artifacts.
      *
      * @param origFile Original case file
      * @param resultDir Output directory of ESEDatabaseViewer
@@ -444,7 +451,9 @@ final class ExtractEdge extends Extract {
     }
 
     /**
-     * Queries for download files and adds artifacts
+     * Queries for download files and adds artifacts.
+     * 
+     * Leaving for future use.
      * 
      * @param origFile Original case file
      * @param resultDir Output directory of ESEDatabaseViewer
@@ -523,7 +532,7 @@ final class ExtractEdge extends Extract {
     private List<AbstractFile> fetchWebCacheDBFiles() throws TskCoreException {
         org.sleuthkit.autopsy.casemodule.services.FileManager fileManager
                 = currentCase.getServices().getFileManager();
-        return fileManager.findFiles(dataSource, EDGE_WEBCACHE_NAME, "WebCache"); //NON-NLS
+        return fileManager.findFiles(dataSource, EDGE_WEBCACHE_NAME, EDGE_WEBCACHE_FOLDER_NAME);
     }
 
     /**
@@ -535,7 +544,7 @@ final class ExtractEdge extends Extract {
     private List<AbstractFile> fetchSpartanDBFiles() throws TskCoreException {
         org.sleuthkit.autopsy.casemodule.services.FileManager fileManager
                 = currentCase.getServices().getFileManager();
-        return fileManager.findFiles(dataSource, EDGE_SPARTAN_NAME, "MicrosoftEdge"); //NON-NLS
+        return fileManager.findFiles(dataSource, EDGE_SPARTAN_NAME, EDGE_SPARTAN_FOLDER_NAME);
     }
 
     /**
@@ -621,7 +630,7 @@ final class ExtractEdge extends Extract {
      * @throws TskCoreException
      */
     private BlackboardArtifact getCookieArtifact(AbstractFile origFile, List<String> headers, String line) throws TskCoreException {
-        String[] lineSplit = line.split(",");
+        String[] lineSplit = line.split(","); // NON-NLS
 
         String accessTime = lineSplit[headers.indexOf(EDGE_HEAD_LASTMOD)].trim();
         Long ftime = null;
@@ -635,9 +644,10 @@ final class ExtractEdge extends Extract {
         String domain = lineSplit[headers.indexOf(EDGE_HEAD_RDOMAIN)].trim();
         String name = hexToChar(lineSplit[headers.indexOf(EDGE_HEAD_NAME)].trim());
         String value = hexToChar(lineSplit[headers.indexOf(EDGE_HEAD_VALUE)].trim());
+        String url = flipDomain(domain);
 
         BlackboardArtifact bbart = origFile.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_COOKIE);
-        bbart.addAttributes(createCookieAttributes(null, ftime, name, value, this.getName(), flipDomain(domain)));
+        bbart.addAttributes(createCookieAttributes(url, ftime, name, value, this.getName(), NetworkUtils.extractDomain(url)));
         return bbart;
     }
 
@@ -655,18 +665,12 @@ final class ExtractEdge extends Extract {
      * @throws TskCoreException
      */
     private BlackboardArtifact getDownloadArtifact(AbstractFile origFile, List<String> headers, String line) throws TskCoreException {
-
+        BlackboardArtifact bbart = null;
-//        String[] lineSplit = line.split(",");
+        
-//
+        String[] lineSplit = line.split(","); // NON-NLS
-//        String url = lineSplit[headers.indexOf(EDGE_HEAD_URL)];
+        String rheader = lineSplit[headers.indexOf(EDGE_HEAD_RESPONSEHEAD)];
-//
+        
-//        String rheader = lineSplit[headers.indexOf(EDGE_HEAD_RESPONSEHEAD)];
+        return bbart;
-//
-//        String decodedheader = this.hexToASCII(rheader);
-//        BlackboardArtifact bbart = origFile.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_DOWNLOAD);
-//        bbart.addAttributes(createDownloadAttributes(decodedheader, "Test2", null, "microsoft.com", this.getName()));
-//        return bbart;
-        return null;
     }
 
     /**
@@ -686,7 +690,7 @@ final class ExtractEdge extends Extract {
         String[] lineSplit = line.split(IGNORE_COMMA_IN_QUOTES_REGEX, -1);
 
         String url = lineSplit[headers.indexOf(EDGE_HEAD_URL)];
-        String title = lineSplit[headers.indexOf(EDGE_HEAD_TITLE)].replace("\"", "");
+        String title = lineSplit[headers.indexOf(EDGE_HEAD_TITLE)].replace("\"", ""); // NON-NLS
 
         if (url.isEmpty()) {
             return null;
@@ -697,7 +701,7 @@ final class ExtractEdge extends Extract {
                 this.getName(), NetworkUtils.extractDomain(url)));
         return bbart;
     }
-    
+
     /**
      * Converts a space separated string of hex values to ascii characters.
      *
@@ -705,7 +709,7 @@ final class ExtractEdge extends Extract {
      * @return "decoded" string or null if a non-hex value was found
      */
     private String hexToChar(String hexString) {
-        String[] hexValues = hexString.split(" ");
+        String[] hexValues = hexString.split(" "); // NON-NLS
         StringBuilder output = new StringBuilder();
 
         for (String str : hexValues) {
@@ -738,7 +742,7 @@ final class ExtractEdge extends Extract {
             return null;
         }
 
-        String[] tokens = domain.split("\\.");
+        String[] tokens = domain.split("\\."); // NON-NLS
 
         if (tokens.length < 2 || tokens.length > 3) {
             return domain; // don't know what to do, just send it back as is
@@ -830,7 +834,7 @@ final class ExtractEdge extends Extract {
                         nameIdx = headers.indexOf(EDGE_HEAD_NAME);
                         idIdx = headers.indexOf(EDGE_HEAD_CONTAINER_ID);
                     } else {
-                        String[] row = line.split(",");
+                        String[] row = line.split(","); // NON-NLS
                         String name = row[nameIdx];
                         String id = row[idIdx];
 

RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractIE.java

@@ -41,6 +41,7 @@ import java.util.Collection;
 import java.util.Scanner;
 import java.util.stream.Collectors;
 import org.openide.modules.InstalledFileLocator;
+import org.openide.util.NbBundle.Messages;
 import org.sleuthkit.autopsy.casemodule.Case;
 import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException;
 import org.sleuthkit.autopsy.datamodel.ContentUtils;
@@ -53,6 +54,7 @@ import org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE;
 import org.sleuthkit.datamodel.Content;
 import org.sleuthkit.autopsy.coreutils.PlatformUtil;
 import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProcessTerminator;
+import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProgress;
 import org.sleuthkit.autopsy.ingest.IngestJobContext;
 import org.sleuthkit.datamodel.*;
 
@@ -71,6 +73,16 @@ class ExtractIE extends Extract {
     private static final SimpleDateFormat dateFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'");
     private Content dataSource;
     private IngestJobContext context;
+    
+    @Messages({
+        "Progress_Message_IE_History=IE History",
+        "Progress_Message_IE_Bookmarks=IE Bookmarks",
+        "Progress_Message_IE_Cookies=IE Cookies",
+        "Progress_Message_IE_Downloads=IE Downloads",
+        "Progress_Message_IE_FormHistory=IE Form History",
+        "Progress_Message_IE_AutoFill=IE Auto Fill",
+        "Progress_Message_IE_Logins=IE Logins",
+    })
 
     ExtractIE() throws NoCurrentCaseException {
         moduleName = NbBundle.getMessage(ExtractIE.class, "ExtractIE.moduleName.text");
@@ -79,12 +91,18 @@ class ExtractIE extends Extract {
     }
 
     @Override
-    public void process(Content dataSource, IngestJobContext context) {
+    public void process(Content dataSource, IngestJobContext context, DataSourceIngestModuleProgress progressBar) {
         this.dataSource = dataSource;
         this.context = context;
         dataFound = false;
+        
+        progressBar.progress(Bundle.Progress_Message_IE_Bookmarks());
         this.getBookmark();
+        
+        progressBar.progress(Bundle.Progress_Message_IE_Cookies());
         this.getCookie();
+        
+        progressBar.progress(Bundle.Progress_Message_IE_History());
         this.getHistory();
     }
 

RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractOs.java

@@ -28,6 +28,7 @@ import org.apache.commons.io.FilenameUtils;
 import org.openide.util.NbBundle.Messages;
 import org.sleuthkit.autopsy.casemodule.services.FileManager;
 import org.sleuthkit.autopsy.coreutils.Logger;
+import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProgress;
 import org.sleuthkit.autopsy.ingest.IngestJobContext;
 import org.sleuthkit.datamodel.AbstractFile;
 import org.sleuthkit.datamodel.BlackboardArtifact;
@@ -39,7 +40,8 @@ import org.sleuthkit.datamodel.TskCoreException;
  * Create OS INFO artifacts for the Operating Systems believed to be present on
  * the data source.
  */
-@Messages({"ExtractOs.parentModuleName=Recent Activity"})
+@Messages({"ExtractOs.parentModuleName=Recent Activity",
+            "ExtractOS_progressMessage=Checking for OS"})
 class ExtractOs extends Extract {
 
     private static final Logger logger = Logger.getLogger(ExtractOs.class.getName());
@@ -64,9 +66,10 @@ class ExtractOs extends Extract {
     private Content dataSource;
 
     @Override
-    void process(Content dataSource, IngestJobContext context) {
+    void process(Content dataSource, IngestJobContext context, DataSourceIngestModuleProgress progressBar) {
         this.dataSource = dataSource;
         try {
+            progressBar.progress(Bundle.ExtractOS_progressMessage());
             for (OS_TYPE value : OS_TYPE.values()) {
                 checkForOSFiles(value);
             }

RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java

@@ -52,6 +52,7 @@ import org.xml.sax.SAXException;
 import java.nio.file.Path;
 import static java.util.TimeZone.getTimeZone;
 import org.openide.util.Lookup;
+import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProgress;
 import org.sleuthkit.autopsy.ingest.IngestModule.IngestModuleException;
 import org.sleuthkit.autopsy.ingest.IngestServices;
 import org.sleuthkit.autopsy.ingest.ModuleDataEvent;
@@ -66,7 +67,8 @@ import org.sleuthkit.datamodel.ReadContentInputStream.ReadContentInputStreamExce
  */
 @NbBundle.Messages({
     "RegRipperNotFound=Autopsy RegRipper executable not found.",
-    "RegRipperFullNotFound=Full version RegRipper executable not found."
+    "RegRipperFullNotFound=Full version RegRipper executable not found.",
+    "Progress_Message_Analyze_Registry=Analyzing Registry Files"
 })
 class ExtractRegistry extends Extract {
 
@@ -969,9 +971,11 @@ class ExtractRegistry extends Extract {
     }
 
     @Override
-    public void process(Content dataSource, IngestJobContext context) {
+    public void process(Content dataSource, IngestJobContext context, DataSourceIngestModuleProgress progressBar) {
         this.dataSource = dataSource;
         this.context = context;
+        
+        progressBar.progress(Bundle.Progress_Message_Analyze_Registry());
         analyzeRegistryFiles();
 
     }

RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractSafari.java

@@ -41,6 +41,7 @@ import org.sleuthkit.autopsy.casemodule.services.FileManager;
 import org.sleuthkit.autopsy.coreutils.Logger;
 import org.sleuthkit.autopsy.coreutils.NetworkUtils;
 import org.sleuthkit.autopsy.datamodel.ContentUtils;
+import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProgress;
 import org.sleuthkit.autopsy.ingest.IngestJobContext;
 import org.sleuthkit.autopsy.ingest.IngestServices;
 import org.sleuthkit.autopsy.ingest.ModuleDataEvent;
@@ -89,6 +90,10 @@ final class ExtractSafari extends Extract {
         "ExtractSafari_Error_Getting_History=An error occurred while processing Safari history files.",
         "ExtractSafari_Error_Parsing_Bookmark=An error occured while processing Safari Bookmark files",
         "ExtractSafari_Error_Parsing_Cookies=An error occured while processing Safari Cookies files",
+        "Progress_Message_Safari_History=Safari History",
+        "Progress_Message_Safari_Bookmarks=Safari Bookmarks",
+        "Progress_Message_Safari_Cookies=Safari Cookies",
+        "Progress_Message_Safari_Downloads=Safari Downloads",
     })
 
     /**
@@ -105,9 +110,10 @@ final class ExtractSafari extends Extract {
     }
 
     @Override
-    void process(Content dataSource, IngestJobContext context) {
+    void process(Content dataSource, IngestJobContext context, DataSourceIngestModuleProgress progressBar) {
         setFoundData(false);
-
+        
+        progressBar.progress(Bundle.Progress_Message_Safari_Cookies());
         try {
             processHistoryDB(dataSource, context);
 
@@ -116,6 +122,7 @@ final class ExtractSafari extends Extract {
             LOG.log(Level.SEVERE, "Exception thrown while processing history file: {0}", ex); //NON-NLS
         }
 
+        progressBar.progress(Bundle.Progress_Message_Safari_Bookmarks());
         try {
             processBookmarkPList(dataSource, context);
         } catch (IOException | TskCoreException | SAXException | PropertyListFormatException | ParseException | ParserConfigurationException ex) {
@@ -123,13 +130,15 @@ final class ExtractSafari extends Extract {
             LOG.log(Level.SEVERE, "Exception thrown while parsing Safari Bookmarks file: {0}", ex); //NON-NLS
         }
         
-         try {
+        progressBar.progress(Bundle.Progress_Message_Safari_Downloads());
+        try {
             processDownloadsPList(dataSource, context);
         } catch (IOException | TskCoreException | SAXException | PropertyListFormatException | ParseException | ParserConfigurationException ex) {
             this.addErrorMessage(Bundle.ExtractSafari_Error_Parsing_Bookmark());
             LOG.log(Level.SEVERE, "Exception thrown while parsing Safari Download.plist file: {0}", ex); //NON-NLS
         }
 
+        progressBar.progress(Bundle.Progress_Message_Safari_Cookies());
         try {
             processBinaryCookieFile(dataSource, context);
         } catch (IOException | TskCoreException ex) {

RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Firefox.java

@@ -44,12 +44,14 @@ import java.util.Set;
 import java.util.logging.Level;
 
 import org.openide.util.NbBundle;
+import org.openide.util.NbBundle.Messages;
 import org.sleuthkit.autopsy.casemodule.Case;
 import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException;
 import org.sleuthkit.autopsy.casemodule.services.FileManager;
 import org.sleuthkit.autopsy.coreutils.Logger;
 import org.sleuthkit.autopsy.coreutils.NetworkUtils;
 import org.sleuthkit.autopsy.datamodel.ContentUtils;
+import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProgress;
 import org.sleuthkit.autopsy.ingest.IngestJobContext;
 import org.sleuthkit.autopsy.ingest.IngestServices;
 import org.sleuthkit.autopsy.ingest.ModuleDataEvent;
@@ -63,6 +65,15 @@ import org.sleuthkit.datamodel.Content;
 import org.sleuthkit.datamodel.ReadContentInputStream.ReadContentInputStreamException;
 import org.sleuthkit.datamodel.TskCoreException;
 
+@Messages({
+    "Progress_Message_Firefox_History=Firefox History",
+    "Progress_Message_Firefox_Bookmarks=Firefox Bookmarks",
+    "Progress_Message_Firefox_Cookies=Firefox Cookies",
+    "Progress_Message_Firefox_Downloads=Firefox Downloads",
+    "Progress_Message_Firefox_FormHistory=Firefox Form History",
+    "Progress_Message_Firefox_AutoFill=Firefox Auto Fill"
+})
+
 /**
  * Firefox recent activity extraction
  */
@@ -95,15 +106,27 @@ class Firefox extends Extract {
     }
 
     @Override
-    public void process(Content dataSource, IngestJobContext context) {
+    public void process(Content dataSource, IngestJobContext context, DataSourceIngestModuleProgress progressBar) {
         this.dataSource = dataSource;
         this.context = context;
         dataFound = false;
+        
+        progressBar.progress(Bundle.Progress_Message_Firefox_History());
         this.getHistory();
+        
+        progressBar.progress(Bundle.Progress_Message_Firefox_Bookmarks());
         this.getBookmark();
+        
+        progressBar.progress(Bundle.Progress_Message_Firefox_Downloads());
         this.getDownload();
+        
+        progressBar.progress(Bundle.Progress_Message_Firefox_Cookies());
         this.getCookie();
+        
+        progressBar.progress(Bundle.Progress_Message_Firefox_FormHistory());
         this.getFormsHistory();
+        
+        progressBar.progress(Bundle.Progress_Message_Firefox_AutoFill());
         this.getAutofillProfiles();
     }
 

RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RAImageIngestModule.java

@@ -120,7 +120,7 @@ public final class RAImageIngestModule implements DataSourceIngestModule {
             progressBar.progress(extracter.getName(), i);
 
             try {
-                extracter.process(dataSource, context);
+                extracter.process(dataSource, context, progressBar);
             } catch (Exception ex) {
                 logger.log(Level.SEVERE, "Exception occurred in " + extracter.getName(), ex); //NON-NLS
                 subCompleted.append(NbBundle.getMessage(this.getClass(), "RAImageIngestModule.process.errModFailed",

RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RecentDocumentsByLnk.java

@@ -29,6 +29,7 @@ import java.util.logging.Level;
 import org.openide.util.NbBundle;
 import org.sleuthkit.autopsy.coreutils.Logger;
 import java.util.Collection;
+import org.openide.util.NbBundle.Messages;
 import org.sleuthkit.autopsy.coreutils.JLNK;
 import org.sleuthkit.autopsy.coreutils.JLnkParser;
 import org.sleuthkit.autopsy.coreutils.JLnkParserException;
@@ -53,6 +54,10 @@ class RecentDocumentsByLnk extends Extract {
     private IngestServices services = IngestServices.getInstance();
     private Content dataSource;
     private IngestJobContext context;
+    
+    @Messages({
+        "Progress_Message_Extract_Resent_Docs=Recent Documents",
+    })
 
     /**
      * Find the documents that Windows stores about recent documents and make
@@ -125,10 +130,12 @@ class RecentDocumentsByLnk extends Extract {
     }
 
     @Override
-    public void process(Content dataSource, IngestJobContext context) {
+    public void process(Content dataSource, IngestJobContext context, DataSourceIngestModuleProgress progressBar) {
         this.dataSource = dataSource;
         this.context = context;
         dataFound = false;
+        
+        progressBar.progress(Bundle.Progress_Message_Extract_Resent_Docs());
         this.getRecentDocuments();
     }
 }

RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java

@@ -33,6 +33,7 @@ import org.openide.util.NbBundle;
 import org.sleuthkit.autopsy.coreutils.Logger;
 import org.sleuthkit.autopsy.coreutils.PlatformUtil;
 import org.sleuthkit.autopsy.coreutils.XMLUtil;
+import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProgress;
 import org.sleuthkit.autopsy.ingest.IngestJobContext;
 import org.sleuthkit.autopsy.ingest.IngestModule.IngestModuleException;
 import org.sleuthkit.autopsy.ingest.IngestServices;
@@ -62,7 +63,8 @@ import org.xml.sax.SAXException;
     "cannotBuildXmlParser=Unable to build XML parser: ",
     "cannotLoadSEUQA=Unable to load Search Engine URL Query Analyzer settings file, SEUQAMappings.xml: ",
     "cannotParseXml=Unable to parse XML file: ",
-    "# {0} - file name", "SearchEngineURLQueryAnalyzer.init.exception.msg=Unable to find {0}."
+    "# {0} - file name", "SearchEngineURLQueryAnalyzer.init.exception.msg=Unable to find {0}.",
+    "Progress_Message_Find_Search_Query=Find Search Queries"
 })
 class SearchEngineURLQueryAnalyzer extends Extract {
 
@@ -396,9 +398,11 @@ class SearchEngineURLQueryAnalyzer extends Extract {
     }
 
     @Override
-    public void process(Content dataSource, IngestJobContext context) {
+    public void process(Content dataSource, IngestJobContext context, DataSourceIngestModuleProgress progressBar) {
         this.dataSource = dataSource;
         this.context = context;
+        
+        progressBar.progress(Bundle.Progress_Message_Find_Search_Query());
         this.findSearchQueries();
         logger.log(Level.INFO, "Search Engine stats: \n{0}", getTotals()); //NON-NLS
     }