diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/contentviewer/DataContentViewerOtherCases.java b/Core/src/org/sleuthkit/autopsy/centralrepository/contentviewer/DataContentViewerOtherCases.java index 4762da103a..308e14be63 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/contentviewer/DataContentViewerOtherCases.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/contentviewer/DataContentViewerOtherCases.java @@ -380,7 +380,7 @@ public class DataContentViewerOtherCases extends JPanel implements DataContentVi * a case to be unique. We should improve this in the future. */ Set cases = new HashSet<>(); - Map devices = new HashMap(); + Map devices = new HashMap<>(); for (int i=0; i < model.getRowCount(); i++) { String caseName = (String) model.getValueAt(i, caseColumnIndex); diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java index df30f3f5f8..5dc8bff71f 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java @@ -62,7 +62,7 @@ abstract class AbstractSqlEamDb implements EamDb { static final String SCHEMA_MINOR_VERSION_KEY = "SCHEMA_MINOR_VERSION"; static final String CREATION_SCHEMA_MAJOR_VERSION_KEY = "CREATION_SCHEMA_MAJOR_VERSION"; static final String CREATION_SCHEMA_MINOR_VERSION_KEY = "CREATION_SCHEMA_MINOR_VERSION"; - static final CaseDbSchemaVersionNumber SOFTWARE_CR_DB_SCHEMA_VERSION = new CaseDbSchemaVersionNumber(1, 2); + static final CaseDbSchemaVersionNumber SOFTWARE_CR_DB_SCHEMA_VERSION = new CaseDbSchemaVersionNumber(1, 3); protected final List defaultCorrelationTypes; @@ -106,6 +106,11 @@ abstract class AbstractSqlEamDb implements EamDb { }); } + /** + * Setup and create a connection to the selected database implementation + */ + protected abstract Connection connect(boolean foreignKeys) throws EamDbException; + /** * Setup and create a connection to the selected database implementation */ @@ -3382,12 +3387,13 @@ abstract class AbstractSqlEamDb implements EamDb { Statement statement = null; PreparedStatement preparedStatement = null; Connection conn = null; + EamDbPlatformEnum selectedPlatform = null; try { - conn = connect(); + conn = connect(false); conn.setAutoCommit(false); statement = conn.createStatement(); - + selectedPlatform = EamDbPlatformEnum.getSelectedPlatform(); int minorVersion = 0; String minorVersionStr = null; resultSet = statement.executeQuery("SELECT value FROM db_info WHERE name='" + AbstractSqlEamDb.SCHEMA_MINOR_VERSION_KEY + "'"); @@ -3441,8 +3447,6 @@ abstract class AbstractSqlEamDb implements EamDb { return; } - EamDbPlatformEnum selectedPlatform = EamDbPlatformEnum.getSelectedPlatform(); - /* * Update to 1.1 */ @@ -3626,7 +3630,36 @@ abstract class AbstractSqlEamDb implements EamDb { statement.execute("INSERT INTO db_info (name, value) VALUES ('" + AbstractSqlEamDb.CREATION_SCHEMA_MAJOR_VERSION_KEY + "','" + creationMajorVer + "')"); statement.execute("INSERT INTO db_info (name, value) VALUES ('" + AbstractSqlEamDb.CREATION_SCHEMA_MINOR_VERSION_KEY + "','" + creationMinorVer + "')"); } + /* + * Update to 1.3 + */ + if (dbSchemaVersion.compareTo(new CaseDbSchemaVersionNumber(1, 3)) < 0) { + switch (selectedPlatform) { + case POSTGRESQL: + statement.execute("ALTER TABLE data_sources DROP CONSTRAINT datasource_unique"); + //unique constraint for upgraded data_sources table is purposefully different than new data_sources table + statement.execute("ALTER TABLE data_sources ADD CONSTRAINT datasource_unique UNIQUE (case_id, device_id, name, datasource_obj_id)"); + break; + case SQLITE: + statement.execute("DROP INDEX IF EXISTS data_sources_name"); + statement.execute("DROP INDEX IF EXISTS data_sources_object_id"); + statement.execute("ALTER TABLE data_sources RENAME TO old_data_sources"); + //unique constraint for upgraded data_sources table is purposefully different than new data_sources table + statement.execute("CREATE TABLE IF NOT EXISTS data_sources (id integer primary key autoincrement NOT NULL," + + "case_id integer NOT NULL,device_id text NOT NULL,name text NOT NULL,datasource_obj_id integer," + + "md5 text DEFAULT NULL,sha1 text DEFAULT NULL,sha256 text DEFAULT NULL," + + "foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL," + + "CONSTRAINT datasource_unique UNIQUE (case_id, device_id, name, datasource_obj_id))"); + statement.execute(SqliteEamDbSettings.getAddDataSourcesNameIndexStatement()); + statement.execute(SqliteEamDbSettings.getAddDataSourcesObjectIdIndexStatement()); + statement.execute("INSERT INTO data_sources SELECT * FROM old_data_sources"); + statement.execute("DROP TABLE old_data_sources"); + break; + default: + throw new EamDbException("Currently selected database platform \"" + selectedPlatform.name() + "\" can not be upgraded."); + } + } updateSchemaVersion(conn); conn.commit(); logger.log(Level.INFO, String.format("Central Repository schema updated to version %s", SOFTWARE_CR_DB_SCHEMA_VERSION)); diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDb.java index 769b49bfd3..89382289f0 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDb.java @@ -161,6 +161,21 @@ final class PostgresEamDb extends AbstractSqlEamDb { connectionPool.setValidationQuery(dbSettings.getValidationQuery()); } + /** + * Lazily setup Singleton connection on first request. + * + * @param foreignKeys -ignored arguement with postgres databases + * + * @return A connection from the connection pool. + * + * @throws EamDbException + */ + @Override + protected Connection connect(boolean foreignKeys) throws EamDbException { + //foreignKeys boolean is ignored for postgres + return connect(); + } + /** * Lazily setup Singleton connection on first request. * @@ -179,7 +194,6 @@ final class PostgresEamDb extends AbstractSqlEamDb { setupConnectionPool(); } } - try { return connectionPool.getConnection(); } catch (SQLException ex) { diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDbSettings.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDbSettings.java index c72cc5f58f..a4ab17906b 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDbSettings.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDbSettings.java @@ -1,7 +1,7 @@ /* * Central Repository * - * Copyright 2015-2017 Basis Technology Corp. + * Copyright 2015-2019 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -43,16 +43,16 @@ import static org.sleuthkit.autopsy.centralrepository.datamodel.AbstractSqlEamDb public final class PostgresEamDbSettings { private final static Logger LOGGER = Logger.getLogger(PostgresEamDbSettings.class.getName()); - private final String DEFAULT_HOST = ""; // NON-NLS - private final int DEFAULT_PORT = 5432; - private final String DEFAULT_DBNAME = "central_repository"; // NON-NLS - private final String DEFAULT_USERNAME = ""; - private final String DEFAULT_PASSWORD = ""; - private final String VALIDATION_QUERY = "SELECT version()"; // NON-NLS - private final String JDBC_BASE_URI = "jdbc:postgresql://"; // NON-NLS - private final String JDBC_DRIVER = "org.postgresql.Driver"; // NON-NLS - private final String DB_NAMES_REGEX = "[a-z][a-z0-9_]*"; // only lower case - private final String DB_USER_NAMES_REGEX = "[a-zA-Z]\\w*"; + private final static String DEFAULT_HOST = ""; // NON-NLS + private final static int DEFAULT_PORT = 5432; + private final static String DEFAULT_DBNAME = "central_repository"; // NON-NLS + private final static String DEFAULT_USERNAME = ""; + private final static String DEFAULT_PASSWORD = ""; + private final static String VALIDATION_QUERY = "SELECT version()"; // NON-NLS + private final static String JDBC_BASE_URI = "jdbc:postgresql://"; // NON-NLS + private final static String JDBC_DRIVER = "org.postgresql.Driver"; // NON-NLS + private final static String DB_NAMES_REGEX = "[a-z][a-z0-9_]*"; // only lower case + private final static String DB_USER_NAMES_REGEX = "[a-zA-Z]\\w*"; private String host; private int port; private String dbName; @@ -339,23 +339,6 @@ public final class PostgresEamDbSettings { String casesIdx1 = "CREATE INDEX IF NOT EXISTS cases_org_id ON cases (org_id)"; String casesIdx2 = "CREATE INDEX IF NOT EXISTS cases_case_uid ON cases (case_uid)"; - StringBuilder createDataSourcesTable = new StringBuilder(); - createDataSourcesTable.append("CREATE TABLE IF NOT EXISTS data_sources ("); - createDataSourcesTable.append("id SERIAL PRIMARY KEY,"); - createDataSourcesTable.append("case_id integer NOT NULL,"); - createDataSourcesTable.append("device_id text NOT NULL,"); - createDataSourcesTable.append("name text NOT NULL,"); - createDataSourcesTable.append("datasource_obj_id BIGINT,"); - createDataSourcesTable.append("md5 text DEFAULT NULL,"); - createDataSourcesTable.append("sha1 text DEFAULT NULL,"); - createDataSourcesTable.append("sha256 text DEFAULT NULL,"); - createDataSourcesTable.append("foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL,"); - createDataSourcesTable.append("CONSTRAINT datasource_unique UNIQUE (case_id, device_id, name)"); - createDataSourcesTable.append(")"); - - String dataSourceIdx1 = "CREATE INDEX IF NOT EXISTS data_sources_name ON data_sources (name)"; - String dataSourceIdx2 = "CREATE INDEX IF NOT EXISTS data_sources_object_id ON data_sources (datasource_obj_id)"; - StringBuilder createReferenceSetsTable = new StringBuilder(); createReferenceSetsTable.append("CREATE TABLE IF NOT EXISTS reference_sets ("); createReferenceSetsTable.append("id SERIAL PRIMARY KEY,"); @@ -422,10 +405,10 @@ public final class PostgresEamDbSettings { stmt.execute(casesIdx1); stmt.execute(casesIdx2); - stmt.execute(createDataSourcesTable.toString()); - stmt.execute(dataSourceIdx1); - stmt.execute(dataSourceIdx2); - + stmt.execute(getCreateDataSourcesTableStatement()); + stmt.execute(getAddDataSourcesNameIndexStatement()); + stmt.execute(getAddDataSourcesObjectIdIndexStatement()); + stmt.execute(createReferenceSetsTable.toString()); stmt.execute(referenceSetsIdx1); @@ -487,21 +470,50 @@ public final class PostgresEamDbSettings { */ static String getCreateArtifactInstancesTableTemplate() { // Each "%s" will be replaced with the relevant TYPE_instances table name. - StringBuilder createArtifactInstancesTableTemplate = new StringBuilder(); - createArtifactInstancesTableTemplate.append("CREATE TABLE IF NOT EXISTS %s ("); - createArtifactInstancesTableTemplate.append("id SERIAL PRIMARY KEY,"); - createArtifactInstancesTableTemplate.append("case_id integer NOT NULL,"); - createArtifactInstancesTableTemplate.append("data_source_id integer NOT NULL,"); - createArtifactInstancesTableTemplate.append("value text NOT NULL,"); - createArtifactInstancesTableTemplate.append("file_path text NOT NULL,"); - createArtifactInstancesTableTemplate.append("known_status integer NOT NULL,"); - createArtifactInstancesTableTemplate.append("comment text,"); - createArtifactInstancesTableTemplate.append("file_obj_id BIGINT,"); - createArtifactInstancesTableTemplate.append("CONSTRAINT %s_multi_unique_ UNIQUE (data_source_id, value, file_path),"); - createArtifactInstancesTableTemplate.append("foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL,"); - createArtifactInstancesTableTemplate.append("foreign key (data_source_id) references data_sources(id) ON UPDATE SET NULL ON DELETE SET NULL"); - createArtifactInstancesTableTemplate.append(")"); - return createArtifactInstancesTableTemplate.toString(); + return ("CREATE TABLE IF NOT EXISTS %s (id SERIAL PRIMARY KEY,case_id integer NOT NULL," + + "data_source_id integer NOT NULL,value text NOT NULL,file_path text NOT NULL," + + "known_status integer NOT NULL,comment text,file_obj_id BIGINT," + + "CONSTRAINT %s_multi_unique_ UNIQUE (data_source_id, value, file_path)," + + "foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL," + + "foreign key (data_source_id) references data_sources(id) ON UPDATE SET NULL ON DELETE SET NULL)"); + } + + /** + * Get the statement String for creating a new data_sources table in a + * Postgres central repository. + * + * @return a String which is a statement for cretating a new data_sources + * table + */ + static String getCreateDataSourcesTableStatement() { + return "CREATE TABLE IF NOT EXISTS data_sources " + + "(id SERIAL PRIMARY KEY,case_id integer NOT NULL,device_id text NOT NULL," + + "name text NOT NULL,datasource_obj_id BIGINT,md5 text DEFAULT NULL," + + "sha1 text DEFAULT NULL,sha256 text DEFAULT NULL," + + "foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL," + + "CONSTRAINT datasource_unique UNIQUE (case_id, datasource_obj_id))"; + } + + /** + * Get the statement for creating an index on the name column of the + * data_sources table. + * + * @return a String which is a statement for adding an index on the name + * column of the data_sources table. + */ + static String getAddDataSourcesNameIndexStatement() { + return "CREATE INDEX IF NOT EXISTS data_sources_name ON data_sources (name)"; + } + + /** + * Get the statement for creating an index on the data_sources_object_id + * column of the data_sources table. + * + * @return a String which is a statement for adding an index on the + * data_sources_object_id column of the data_sources table. + */ + static String getAddDataSourcesObjectIdIndexStatement() { + return "CREATE INDEX IF NOT EXISTS data_sources_object_id ON data_sources (datasource_obj_id)"; } /** @@ -561,8 +573,8 @@ public final class PostgresEamDbSettings { * instance table. %s will exist in the template where the name of the new * table will be addedd. * - * @return a String which is a template for adding an index to the file_obj_id - * column of a _instances table + * @return a String which is a template for adding an index to the + * file_obj_id column of a _instances table */ static String getAddObjectIdIndexTemplate() { // Each "%s" will be replaced with the relevant TYPE_instances table name. diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDb.java index 9e486855bb..492d13dfa6 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDb.java @@ -153,7 +153,7 @@ final class SqliteEamDb extends AbstractSqlEamDb { * Setup a connection pool for db connections. * */ - private void setupConnectionPool() throws EamDbException { + private void setupConnectionPool(boolean foreignKeysEnabled) throws EamDbException { if (dbSettings.dbFileExists() == false) { throw new EamDbException("Central repository database missing"); @@ -169,27 +169,31 @@ final class SqliteEamDb extends AbstractSqlEamDb { connectionPool.setMaxIdle(-1); connectionPool.setMaxWaitMillis(1000); connectionPool.setValidationQuery(dbSettings.getValidationQuery()); - connectionPool.setConnectionInitSqls(Arrays.asList("PRAGMA foreign_keys = ON")); + if (foreignKeysEnabled) { + connectionPool.setConnectionInitSqls(Arrays.asList("PRAGMA foreign_keys = ON")); + } else { + connectionPool.setConnectionInitSqls(Arrays.asList("PRAGMA foreign_keys = OFF")); + } } /** * Lazily setup Singleton connection on first request. * + * @param foreignKeys determines if foreign keys should be enforced during this connection for SQLite + * * @return A connection from the connection pool. * * @throws EamDbException */ @Override - protected Connection connect() throws EamDbException { + protected Connection connect(boolean foreignKeys) throws EamDbException { synchronized (this) { if (!EamDb.isEnabled()) { throw new EamDbException("Central Repository module is not enabled"); // NON-NLS } - if (connectionPool == null) { - setupConnectionPool(); + setupConnectionPool(foreignKeys); } - try { return connectionPool.getConnection(); } catch (SQLException ex) { @@ -198,6 +202,18 @@ final class SqliteEamDb extends AbstractSqlEamDb { } } + /** + * Lazily setup Singleton connection on first request with foreign keys enforced. + * + * @return A connection from the connection pool. + * + * @throws EamDbException + */ + @Override + protected Connection connect() throws EamDbException { + return connect(true); + } + @Override protected String getConflictClause() { // For sqlite, our conflict clause is part of the table schema diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDbSettings.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDbSettings.java index 4baa423a04..c76ab19783 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDbSettings.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDbSettings.java @@ -1,7 +1,7 @@ /* * Central Repository * - * Copyright 2015-2017 Basis Technology Corp. + * Copyright 2015-2019 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -43,19 +43,19 @@ import static org.sleuthkit.autopsy.centralrepository.datamodel.AbstractSqlEamDb public final class SqliteEamDbSettings { private final static Logger LOGGER = Logger.getLogger(SqliteEamDbSettings.class.getName()); - private final String DEFAULT_DBNAME = "central_repository.db"; // NON-NLS - private final String DEFAULT_DBDIRECTORY = PlatformUtil.getUserDirectory() + File.separator + "central_repository"; // NON-NLS - private final String JDBC_DRIVER = "org.sqlite.JDBC"; // NON-NLS - private final String JDBC_BASE_URI = "jdbc:sqlite:"; // NON-NLS - private final String VALIDATION_QUERY = "SELECT count(*) from sqlite_master"; // NON-NLS - private static final String PRAGMA_SYNC_OFF = "PRAGMA synchronous = OFF"; - private static final String PRAGMA_SYNC_NORMAL = "PRAGMA synchronous = NORMAL"; - private static final String PRAGMA_JOURNAL_WAL = "PRAGMA journal_mode = WAL"; - private static final String PRAGMA_READ_UNCOMMITTED_TRUE = "PRAGMA read_uncommitted = True"; - private static final String PRAGMA_ENCODING_UTF8 = "PRAGMA encoding = 'UTF-8'"; - private static final String PRAGMA_PAGE_SIZE_4096 = "PRAGMA page_size = 4096"; - private static final String PRAGMA_FOREIGN_KEYS_ON = "PRAGMA foreign_keys = ON"; - private final String DB_NAMES_REGEX = "[a-z][a-z0-9_]*(\\.db)?"; + private final static String DEFAULT_DBNAME = "central_repository.db"; // NON-NLS + private final static String DEFAULT_DBDIRECTORY = PlatformUtil.getUserDirectory() + File.separator + "central_repository"; // NON-NLS + private final static String JDBC_DRIVER = "org.sqlite.JDBC"; // NON-NLS + private final static String JDBC_BASE_URI = "jdbc:sqlite:"; // NON-NLS + private final static String VALIDATION_QUERY = "SELECT count(*) from sqlite_master"; // NON-NLS + private final static String PRAGMA_SYNC_OFF = "PRAGMA synchronous = OFF"; + private final static String PRAGMA_SYNC_NORMAL = "PRAGMA synchronous = NORMAL"; + private final static String PRAGMA_JOURNAL_WAL = "PRAGMA journal_mode = WAL"; + private final static String PRAGMA_READ_UNCOMMITTED_TRUE = "PRAGMA read_uncommitted = True"; + private final static String PRAGMA_ENCODING_UTF8 = "PRAGMA encoding = 'UTF-8'"; + private final static String PRAGMA_PAGE_SIZE_4096 = "PRAGMA page_size = 4096"; + private final static String PRAGMA_FOREIGN_KEYS_ON = "PRAGMA foreign_keys = ON"; + private final static String DB_NAMES_REGEX = "[a-z][a-z0-9_]*(\\.db)?"; private String dbName; private String dbDirectory; private int bulkThreshold; @@ -282,23 +282,6 @@ public final class SqliteEamDbSettings { String casesIdx1 = "CREATE INDEX IF NOT EXISTS cases_org_id ON cases (org_id)"; String casesIdx2 = "CREATE INDEX IF NOT EXISTS cases_case_uid ON cases (case_uid)"; - StringBuilder createDataSourcesTable = new StringBuilder(); - createDataSourcesTable.append("CREATE TABLE IF NOT EXISTS data_sources ("); - createDataSourcesTable.append("id integer primary key autoincrement NOT NULL,"); - createDataSourcesTable.append("case_id integer NOT NULL,"); - createDataSourcesTable.append("device_id text NOT NULL,"); - createDataSourcesTable.append("name text NOT NULL,"); - createDataSourcesTable.append("datasource_obj_id integer,"); - createDataSourcesTable.append("md5 text DEFAULT NULL,"); - createDataSourcesTable.append("sha1 text DEFAULT NULL,"); - createDataSourcesTable.append("sha256 text DEFAULT NULL,"); - createDataSourcesTable.append("foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL,"); - createDataSourcesTable.append("CONSTRAINT datasource_unique UNIQUE (case_id, device_id, name)"); - createDataSourcesTable.append(")"); - - String dataSourceIdx1 = "CREATE INDEX IF NOT EXISTS data_sources_name ON data_sources (name)"; - String dataSourceIdx2 = "CREATE INDEX IF NOT EXISTS data_sources_object_id ON data_sources (datasource_obj_id)"; - StringBuilder createReferenceSetsTable = new StringBuilder(); createReferenceSetsTable.append("CREATE TABLE IF NOT EXISTS reference_sets ("); createReferenceSetsTable.append("id integer primary key autoincrement NOT NULL,"); @@ -371,9 +354,9 @@ public final class SqliteEamDbSettings { stmt.execute(casesIdx1); stmt.execute(casesIdx2); - stmt.execute(createDataSourcesTable.toString()); - stmt.execute(dataSourceIdx1); - stmt.execute(dataSourceIdx2); + stmt.execute(getCreateDataSourcesTableStatement()); + stmt.execute(getAddDataSourcesNameIndexStatement()); + stmt.execute(getAddDataSourcesObjectIdIndexStatement()); stmt.execute(createReferenceSetsTable.toString()); stmt.execute(referenceSetsIdx1); @@ -435,21 +418,49 @@ public final class SqliteEamDbSettings { */ static String getCreateArtifactInstancesTableTemplate() { // Each "%s" will be replaced with the relevant TYPE_instances table name. - StringBuilder createArtifactInstancesTableTemplate = new StringBuilder(); - createArtifactInstancesTableTemplate.append("CREATE TABLE IF NOT EXISTS %s ("); - createArtifactInstancesTableTemplate.append("id integer primary key autoincrement NOT NULL,"); - createArtifactInstancesTableTemplate.append("case_id integer NOT NULL,"); - createArtifactInstancesTableTemplate.append("data_source_id integer NOT NULL,"); - createArtifactInstancesTableTemplate.append("value text NOT NULL,"); - createArtifactInstancesTableTemplate.append("file_path text NOT NULL,"); - createArtifactInstancesTableTemplate.append("known_status integer NOT NULL,"); - createArtifactInstancesTableTemplate.append("comment text,"); - createArtifactInstancesTableTemplate.append("file_obj_id integer,"); - createArtifactInstancesTableTemplate.append("CONSTRAINT %s_multi_unique UNIQUE(data_source_id, value, file_path) ON CONFLICT IGNORE,"); - createArtifactInstancesTableTemplate.append("foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL,"); - createArtifactInstancesTableTemplate.append("foreign key (data_source_id) references data_sources(id) ON UPDATE SET NULL ON DELETE SET NULL"); - createArtifactInstancesTableTemplate.append(")"); - return createArtifactInstancesTableTemplate.toString(); + return "CREATE TABLE IF NOT EXISTS %s (id integer primary key autoincrement NOT NULL," + + "case_id integer NOT NULL,data_source_id integer NOT NULL,value text NOT NULL," + + "file_path text NOT NULL,known_status integer NOT NULL,comment text,file_obj_id integer," + + "CONSTRAINT %s_multi_unique UNIQUE(data_source_id, value, file_path) ON CONFLICT IGNORE," + + "foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL," + + "foreign key (data_source_id) references data_sources(id) ON UPDATE SET NULL ON DELETE SET NULL)"; + } + + /** + * Get the statement String for creating a new data_sources table in a + * Sqlite central repository. + * + * @return a String which is a statement for cretating a new data_sources + * table + */ + static String getCreateDataSourcesTableStatement() { + return "CREATE TABLE IF NOT EXISTS data_sources (id integer primary key autoincrement NOT NULL," + + "case_id integer NOT NULL,device_id text NOT NULL,name text NOT NULL,datasource_obj_id integer," + + "md5 text DEFAULT NULL,sha1 text DEFAULT NULL,sha256 text DEFAULT NULL," + + "foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL," + + "CONSTRAINT datasource_unique UNIQUE (case_id, datasource_obj_id))"; + } + + /** + * Get the statement for creating an index on the name column of the + * data_sources table. + * + * @return a String which is a statement for adding an index on the name + * column of the data_sources table. + */ + static String getAddDataSourcesNameIndexStatement() { + return "CREATE INDEX IF NOT EXISTS data_sources_name ON data_sources (name)"; + } + + /** + * Get the statement for creating an index on the data_sources_object_id + * column of the data_sources table. + * + * @return a String which is a statement for adding an index on the + * data_sources_object_id column of the data_sources table. + */ + static String getAddDataSourcesObjectIdIndexStatement() { + return "CREATE INDEX IF NOT EXISTS data_sources_object_id ON data_sources (datasource_obj_id)"; } /** diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties index 9d82d32924..67e6496584 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties @@ -1,5 +1,5 @@ OpenIDE-Module-Display-Category=Ingest Module -OpenIDE-Module-Long-Description=Keyword Search ingest module.\n\n\The module indexes files found in the disk image at ingest time.\n\It then periodically runs the search on the indexed files using one or more keyword lists (containing pure words and/or regular expressions) and posts results.\n\n\The module also contains additional tools integrated in the main GUI, such as keyword list configuration, keyword seach bar in the top-right corner, extracted text viewer and search results viewer showing highlighted keywords found. +OpenIDE-Module-Long-Description=Keyword Search ingest module.\n\nThe module indexes files found in the disk image at ingest time.\nIt then periodically runs the search on the indexed files using one or more keyword lists (containing pure words and/or regular expressions) and posts results.\n\n\The module also contains additional tools integrated in the main GUI, such as keyword list configuration, keyword search bar in the top-right corner, extracted text viewer and search results viewer showing highlighted keywords found. OpenIDE-Module-Name=KeywordSearch OptionsCategory_Name_KeywordSearchOptions=Keyword Search OptionsCategory_Keywords_KeywordSearchOptions=Keyword Search diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED index 61c0d2d2c7..034e4355d3 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED @@ -34,7 +34,7 @@ KeywordSearchIngestModule.startupMessage.failedToGetIndexSchema=Failed to get sc KeywordSearchResultFactory.createNodeForKey.noResultsFound.text=No results found. KeywordSearchResultFactory.query.exception.msg=Could not perform the query OpenIDE-Module-Display-Category=Ingest Module -OpenIDE-Module-Long-Description=Keyword Search ingest module.\n\nThe module indexes files found in the disk image at ingest time.\nIt then periodically runs the search on the indexed files using one or more keyword lists (containing pure words and/or regular expressions) and posts results.\n\nThe module also contains additional tools integrated in the main GUI, such as keyword list configuration, keyword seach bar in the top-right corner, extracted text viewer and search results viewer showing highlighted keywords found. +OpenIDE-Module-Long-Description=Keyword Search ingest module.\n\nThe module indexes files found in the disk image at ingest time.\nIt then periodically runs the search on the indexed files using one or more keyword lists (containing pure words and/or regular expressions) and posts results.\n\n\The module also contains additional tools integrated in the main GUI, such as keyword list configuration, keyword search bar in the top-right corner, extracted text viewer and search results viewer showing highlighted keywords found. OpenIDE-Module-Name=KeywordSearch OptionsCategory_Name_KeywordSearchOptions=Keyword Search OptionsCategory_Keywords_KeywordSearchOptions=Keyword Search diff --git a/RecentActivity/ivy.xml b/RecentActivity/ivy.xml index ca95f14a98..290c8371ea 100644 --- a/RecentActivity/ivy.xml +++ b/RecentActivity/ivy.xml @@ -6,7 +6,4 @@ - - - diff --git a/RecentActivity/nbproject/project.properties b/RecentActivity/nbproject/project.properties index 4071f4a54e..9736070e53 100644 --- a/RecentActivity/nbproject/project.properties +++ b/RecentActivity/nbproject/project.properties @@ -1,4 +1,3 @@ -file.reference.dd-plist-1.20.jar=release/modules/ext/dd-plist-1.20.jar javac.source=1.8 javac.compilerargs=-Xlint -Xlint:-serial license.file=../LICENSE-2.0.txt diff --git a/RecentActivity/nbproject/project.xml b/RecentActivity/nbproject/project.xml index 9584170602..87619a8356 100644 --- a/RecentActivity/nbproject/project.xml +++ b/RecentActivity/nbproject/project.xml @@ -74,10 +74,6 @@ - - ext/dd-plist-1.20.jar - release/modules/ext/dd-plist-1.20.jar - diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Bundle.properties-MERGED b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Bundle.properties-MERGED index 5b275de006..6c5419d31e 100755 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Bundle.properties-MERGED +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Bundle.properties-MERGED @@ -43,6 +43,7 @@ ExtractOs.unitedLinuxVolume.label=OS Drive (Linux United Linux) ExtractOs.windowsVolume.label=OS Drive (Windows) ExtractOs.yellowDogLinuxOs.label=Linux (Yellow Dog) ExtractOs.yellowDogLinuxVolume.label=OS Drive (Linux Yellow Dog) +ExtractOS_progressMessage=Checking for OS ExtractSafari_Error_Getting_History=An error occurred while processing Safari history files. ExtractSafari_Error_Parsing_Bookmark=An error occured while processing Safari Bookmark files ExtractSafari_Error_Parsing_Cookies=An error occured while processing Safari Cookies files @@ -119,6 +120,37 @@ Firefox.getDlPre24.errMsg.errParsingArtifacts={0}: Error parsing {1} Firefox web Firefox.getDlV24.errMsg.errFetchFiles=Error fetching 'downloads' files for Firefox. Firefox.getDlV24.errMsg.errAnalyzeFile={0}: Error while trying to analyze file:{1} Firefox.getDlV24.errMsg.errParsingArtifacts={0}: Error parsing {1} Firefox web download artifacts. +Progress_Message_Analyze_Registry=Analyzing Registry Files +Progress_Message_Analyze_Usage=Data Sources Usage Analysis +Progress_Message_Chrome_AutoFill=Chrome Auto Fill +Progress_Message_Chrome_Bookmarks=Chrome Bookmarks +Progress_Message_Chrome_Cookies=Chrome Cookies +Progress_Message_Chrome_Downloads=Chrome Downloads +Progress_Message_Chrome_FormHistory=Chrome Form History +Progress_Message_Chrome_History=Chrome History +Progress_Message_Chrome_Logins=Chrome Logins +Progress_Message_Edge_Bookmarks=Microsoft Edge Bookmarks +Progress_Message_Edge_Cookies=Microsoft Edge Cookies +Progress_Message_Edge_History=Microsoft Edge History +Progress_Message_Extract_Resent_Docs=Recent Documents +Progress_Message_Find_Search_Query=Find Search Queries +Progress_Message_Firefox_AutoFill=Firefox Auto Fill +Progress_Message_Firefox_Bookmarks=Firefox Bookmarks +Progress_Message_Firefox_Cookies=Firefox Cookies +Progress_Message_Firefox_Downloads=Firefox Downloads +Progress_Message_Firefox_FormHistory=Firefox Form History +Progress_Message_Firefox_History=Firefox History +Progress_Message_IE_AutoFill=IE Auto Fill +Progress_Message_IE_Bookmarks=IE Bookmarks +Progress_Message_IE_Cookies=IE Cookies +Progress_Message_IE_Downloads=IE Downloads +Progress_Message_IE_FormHistory=IE Form History +Progress_Message_IE_History=IE History +Progress_Message_IE_Logins=IE Logins +Progress_Message_Safari_Bookmarks=Safari Bookmarks +Progress_Message_Safari_Cookies=Safari Cookies +Progress_Message_Safari_Downloads=Safari Downloads +Progress_Message_Safari_History=Safari History RAImageIngestModule.process.started=Started {0} RAImageIngestModule.process.errModFailed={0} failed - see log for details
RAImageIngestModule.process.errModErrs={0} had errors -- see log diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Chrome.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Chrome.java index d97b53426a..503cd639ba 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Chrome.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Chrome.java @@ -37,11 +37,13 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.FileReader; import java.io.IOException; +import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.autopsy.casemodule.services.FileManager; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.NetworkUtils; +import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProgress; import org.sleuthkit.autopsy.ingest.IngestJobContext; import org.sleuthkit.autopsy.ingest.ModuleDataEvent; import org.sleuthkit.datamodel.AbstractFile; @@ -81,21 +83,43 @@ class Chrome extends Extract { private final Logger logger = Logger.getLogger(this.getClass().getName()); private Content dataSource; private IngestJobContext context; + + @Messages({ + "Progress_Message_Chrome_History=Chrome History", + "Progress_Message_Chrome_Bookmarks=Chrome Bookmarks", + "Progress_Message_Chrome_Cookies=Chrome Cookies", + "Progress_Message_Chrome_Downloads=Chrome Downloads", + "Progress_Message_Chrome_FormHistory=Chrome Form History", + "Progress_Message_Chrome_AutoFill=Chrome Auto Fill", + "Progress_Message_Chrome_Logins=Chrome Logins", + }) Chrome() { moduleName = NbBundle.getMessage(Chrome.class, "Chrome.moduleName"); } @Override - public void process(Content dataSource, IngestJobContext context) { + public void process(Content dataSource, IngestJobContext context, DataSourceIngestModuleProgress progressBar) { this.dataSource = dataSource; this.context = context; dataFound = false; + + progressBar.progress(Bundle.Progress_Message_Chrome_History()); this.getHistory(); + + progressBar.progress(Bundle.Progress_Message_Chrome_Bookmarks()); this.getBookmark(); + + progressBar.progress(Bundle.Progress_Message_Chrome_Cookies()); this.getCookie(); + + progressBar.progress(Bundle.Progress_Message_Chrome_Logins()); this.getLogins(); + + progressBar.progress(Bundle.Progress_Message_Chrome_AutoFill()); this.getAutofill(); + + progressBar.progress(Bundle.Progress_Message_Chrome_Downloads()); this.getDownload(); ChromeCacheExtractor chromeCacheExtractor = new ChromeCacheExtractor(dataSource, context); diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/DataSourceUsageAnalyzer.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/DataSourceUsageAnalyzer.java index 9c841442a3..8b40633d2b 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/DataSourceUsageAnalyzer.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/DataSourceUsageAnalyzer.java @@ -26,6 +26,7 @@ import org.apache.commons.io.FilenameUtils; import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.casemodule.services.FileManager; import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProgress; import org.sleuthkit.autopsy.ingest.IngestJobContext; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.BlackboardArtifact; @@ -46,12 +47,14 @@ class DataSourceUsageAnalyzer extends Extract { @Messages({ "# {0} - OS name", - "DataSourceUsageAnalyzer.customVolume.label=OS Drive ({0})" + "DataSourceUsageAnalyzer.customVolume.label=OS Drive ({0})", + "Progress_Message_Analyze_Usage=Data Sources Usage Analysis", }) @Override - void process(Content dataSource, IngestJobContext context) { + void process(Content dataSource, IngestJobContext context, DataSourceIngestModuleProgress progressBar) { this.dataSource = dataSource; try { + progressBar.progress(Bundle.Progress_Message_Analyze_Usage()); createDataSourceUsageArtifacts(); } catch (TskCoreException ex) { logger.log(Level.WARNING, "Failed to check if datasource contained a volume with operating system specific files", ex); diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Extract.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Extract.java index b4f04c79cd..3ffbad514b 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Extract.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Extract.java @@ -44,6 +44,7 @@ import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; import org.sleuthkit.autopsy.coreutils.SQLiteDBConnect; import org.sleuthkit.autopsy.datamodel.ContentUtils; +import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProgress; import org.sleuthkit.autopsy.ingest.IngestJobContext; import org.sleuthkit.autopsy.ingest.IngestModule.IngestModuleException; import org.sleuthkit.datamodel.AbstractFile; @@ -85,7 +86,7 @@ abstract class Extract { void configExtractor() throws IngestModuleException { } - abstract void process(Content dataSource, IngestJobContext context); + abstract void process(Content dataSource, IngestJobContext context, DataSourceIngestModuleProgress progressBar); void complete() { } diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractEdge.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractEdge.java index ff793703b1..c3032e8cc2 100755 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractEdge.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractEdge.java @@ -38,11 +38,13 @@ import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.autopsy.coreutils.ExecUtil; +import org.sleuthkit.autopsy.coreutils.FileUtil; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.NetworkUtils; import org.sleuthkit.autopsy.coreutils.PlatformUtil; import org.sleuthkit.autopsy.datamodel.ContentUtils; import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProcessTerminator; +import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProgress; import org.sleuthkit.autopsy.ingest.IngestJobContext; import org.sleuthkit.autopsy.ingest.IngestServices; import org.sleuthkit.autopsy.ingest.ModuleDataEvent; @@ -94,11 +96,13 @@ final class ExtractEdge extends Extract { private static final String EDGE_FAVORITE_FILE_NAME = "Favorites.csv"; //NON-NLS private static final String EDGE_OUTPUT_FILE_NAME = "Output.txt"; //NON-NLS private static final String EDGE_ERROR_FILE_NAME = "File.txt"; //NON-NLS + private static final String EDGE_WEBCACHE_FOLDER_NAME = "WebCache"; //NON-NLS + private static final String EDGE_SPARTAN_FOLDER_NAME = "MicrosoftEdge"; //NON-NLS private static final String ESE_TOOL_FOLDER = "ESEDatabaseView"; //NON-NLS private static final String EDGE_RESULT_FOLDER_NAME = "results"; //NON-NLS - private static final SimpleDateFormat DATE_FORMATTER = new SimpleDateFormat("MM/dd/yyyy hh:mm:ss a"); + private static final SimpleDateFormat DATE_FORMATTER = new SimpleDateFormat("MM/dd/yyyy hh:mm:ss a"); //NON-NLS @Messages({ "ExtractEdge_process_errMsg_unableFindESEViewer=Unable to find ESEDatabaseViewer", @@ -107,6 +111,9 @@ final class ExtractEdge extends Extract { "ExtractEdge_process_errMsg_spartanFail=Failure processing Microsoft Edge spartan.edb file", "ExtractEdge_Module_Name=Microsoft Edge", "ExtractEdge_getHistory_containerFileNotFound=Error while trying to analyze Edge history", + "Progress_Message_Edge_History=Microsoft Edge History", + "Progress_Message_Edge_Bookmarks=Microsoft Edge Bookmarks", + "Progress_Message_Edge_Cookies=Microsoft Edge Cookies", }) /** @@ -122,7 +129,7 @@ final class ExtractEdge extends Extract { } @Override - void process(Content dataSource, IngestJobContext context) { + void process(Content dataSource, IngestJobContext context, DataSourceIngestModuleProgress progressBar) { this.dataSource = dataSource; this.context = context; this.setFoundData(false); @@ -164,30 +171,31 @@ final class ExtractEdge extends Extract { } try { - this.processWebCacheDbFile(esedumper, webCacheFiles); + this.processWebCacheDbFile(esedumper, webCacheFiles, progressBar); } catch (IOException | TskCoreException ex) { this.addErrorMessage(Bundle.ExtractEdge_process_errMsg_webcacheFail()); LOG.log(Level.SEVERE, "Error returned from processWebCacheDbFile", ex); // NON-NLS } + progressBar.progress(Bundle.Progress_Message_Edge_Bookmarks()); try { this.processSpartanDbFile(esedumper, spartanFiles); } catch (IOException | TskCoreException ex) { - this.addErrorMessage(Bundle.ExtractEdge_process_errMsg_webcacheFail()); + this.addErrorMessage(Bundle.ExtractEdge_process_errMsg_spartanFail()); LOG.log(Level.SEVERE, "Error returned from processSpartanDbFile", ex); // NON-NLS } } /** - * Dump the tables from WebCacheV01.dat and look for the data contained with - * in those files including downloads, cookies and history. + * Process WebCacheV01.dat ese database file creating artifacts for cookies, + * and history contained within. * * @param eseDumperPath Path to ESEDatabaseView.exe * @param webCacheFiles List of case WebCacheV01.dat files * @throws IOException * @throws TskCoreException */ - void processWebCacheDbFile(String eseDumperPath, List webCacheFiles) throws IOException, TskCoreException { + void processWebCacheDbFile(String eseDumperPath, List webCacheFiles, DataSourceIngestModuleProgress progressBar) throws IOException, TskCoreException { for (AbstractFile webCacheFile : webCacheFiles) { @@ -218,30 +226,29 @@ final class ExtractEdge extends Extract { if (context.dataSourceIngestIsCancelled()) { return; } - + + progressBar.progress(Bundle.Progress_Message_Edge_History()); + this.getHistory(webCacheFile, resultsDir); if (context.dataSourceIngestIsCancelled()) { return; } + progressBar.progress(Bundle.Progress_Message_Edge_Cookies()); + this.getCookies(webCacheFile, resultsDir); -// if (context.dataSourceIngestIsCancelled()) { -// return; -// } -// Putting downloads on hold -// this.getDownload(webCacheFile, resultsDir); } finally { tempWebCacheFile.delete(); - resultsDir.delete(); + FileUtil.deleteFileDir(resultsDir); } } } /** - * Creates a temp version of the database and runs the ESEDatabaseView tool - * to dump each of the database tables into a temporary folder. + * Process spartan.edb ese database file creating artifacts for the bookmarks + * contained within. * * @param eseDumperPath Path to ESEDatabaseViewer * @param spartanFiles List of the case spartan.edb files @@ -282,7 +289,7 @@ final class ExtractEdge extends Extract { } finally { tempSpartanFile.delete(); - resultsDir.delete(); + FileUtil.deleteFileDir(resultsDir); } } } @@ -392,7 +399,7 @@ final class ExtractEdge extends Extract { } /** - * Queries for cookie files and adds artifacts + * Queries for cookie files and adds artifacts. * * @param origFile Original case file * @param resultDir Output directory of ESEDatabaseViewer @@ -444,7 +451,9 @@ final class ExtractEdge extends Extract { } /** - * Queries for download files and adds artifacts + * Queries for download files and adds artifacts. + * + * Leaving for future use. * * @param origFile Original case file * @param resultDir Output directory of ESEDatabaseViewer @@ -523,7 +532,7 @@ final class ExtractEdge extends Extract { private List fetchWebCacheDBFiles() throws TskCoreException { org.sleuthkit.autopsy.casemodule.services.FileManager fileManager = currentCase.getServices().getFileManager(); - return fileManager.findFiles(dataSource, EDGE_WEBCACHE_NAME, "WebCache"); //NON-NLS + return fileManager.findFiles(dataSource, EDGE_WEBCACHE_NAME, EDGE_WEBCACHE_FOLDER_NAME); } /** @@ -535,7 +544,7 @@ final class ExtractEdge extends Extract { private List fetchSpartanDBFiles() throws TskCoreException { org.sleuthkit.autopsy.casemodule.services.FileManager fileManager = currentCase.getServices().getFileManager(); - return fileManager.findFiles(dataSource, EDGE_SPARTAN_NAME, "MicrosoftEdge"); //NON-NLS + return fileManager.findFiles(dataSource, EDGE_SPARTAN_NAME, EDGE_SPARTAN_FOLDER_NAME); } /** @@ -621,7 +630,7 @@ final class ExtractEdge extends Extract { * @throws TskCoreException */ private BlackboardArtifact getCookieArtifact(AbstractFile origFile, List headers, String line) throws TskCoreException { - String[] lineSplit = line.split(","); + String[] lineSplit = line.split(","); // NON-NLS String accessTime = lineSplit[headers.indexOf(EDGE_HEAD_LASTMOD)].trim(); Long ftime = null; @@ -635,9 +644,10 @@ final class ExtractEdge extends Extract { String domain = lineSplit[headers.indexOf(EDGE_HEAD_RDOMAIN)].trim(); String name = hexToChar(lineSplit[headers.indexOf(EDGE_HEAD_NAME)].trim()); String value = hexToChar(lineSplit[headers.indexOf(EDGE_HEAD_VALUE)].trim()); + String url = flipDomain(domain); BlackboardArtifact bbart = origFile.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_COOKIE); - bbart.addAttributes(createCookieAttributes(null, ftime, name, value, this.getName(), flipDomain(domain))); + bbart.addAttributes(createCookieAttributes(url, ftime, name, value, this.getName(), NetworkUtils.extractDomain(url))); return bbart; } @@ -655,18 +665,12 @@ final class ExtractEdge extends Extract { * @throws TskCoreException */ private BlackboardArtifact getDownloadArtifact(AbstractFile origFile, List headers, String line) throws TskCoreException { - -// String[] lineSplit = line.split(","); -// -// String url = lineSplit[headers.indexOf(EDGE_HEAD_URL)]; -// -// String rheader = lineSplit[headers.indexOf(EDGE_HEAD_RESPONSEHEAD)]; -// -// String decodedheader = this.hexToASCII(rheader); -// BlackboardArtifact bbart = origFile.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_DOWNLOAD); -// bbart.addAttributes(createDownloadAttributes(decodedheader, "Test2", null, "microsoft.com", this.getName())); -// return bbart; - return null; + BlackboardArtifact bbart = null; + + String[] lineSplit = line.split(","); // NON-NLS + String rheader = lineSplit[headers.indexOf(EDGE_HEAD_RESPONSEHEAD)]; + + return bbart; } /** @@ -686,7 +690,7 @@ final class ExtractEdge extends Extract { String[] lineSplit = line.split(IGNORE_COMMA_IN_QUOTES_REGEX, -1); String url = lineSplit[headers.indexOf(EDGE_HEAD_URL)]; - String title = lineSplit[headers.indexOf(EDGE_HEAD_TITLE)].replace("\"", ""); + String title = lineSplit[headers.indexOf(EDGE_HEAD_TITLE)].replace("\"", ""); // NON-NLS if (url.isEmpty()) { return null; @@ -697,7 +701,7 @@ final class ExtractEdge extends Extract { this.getName(), NetworkUtils.extractDomain(url))); return bbart; } - + /** * Converts a space separated string of hex values to ascii characters. * @@ -705,7 +709,7 @@ final class ExtractEdge extends Extract { * @return "decoded" string or null if a non-hex value was found */ private String hexToChar(String hexString) { - String[] hexValues = hexString.split(" "); + String[] hexValues = hexString.split(" "); // NON-NLS StringBuilder output = new StringBuilder(); for (String str : hexValues) { @@ -738,7 +742,7 @@ final class ExtractEdge extends Extract { return null; } - String[] tokens = domain.split("\\."); + String[] tokens = domain.split("\\."); // NON-NLS if (tokens.length < 2 || tokens.length > 3) { return domain; // don't know what to do, just send it back as is @@ -830,7 +834,7 @@ final class ExtractEdge extends Extract { nameIdx = headers.indexOf(EDGE_HEAD_NAME); idIdx = headers.indexOf(EDGE_HEAD_CONTAINER_ID); } else { - String[] row = line.split(","); + String[] row = line.split(","); // NON-NLS String name = row[nameIdx]; String id = row[idIdx]; diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractIE.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractIE.java index 223ed69799..099579ecae 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractIE.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractIE.java @@ -41,6 +41,7 @@ import java.util.Collection; import java.util.Scanner; import java.util.stream.Collectors; import org.openide.modules.InstalledFileLocator; +import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.autopsy.datamodel.ContentUtils; @@ -53,6 +54,7 @@ import org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE; import org.sleuthkit.datamodel.Content; import org.sleuthkit.autopsy.coreutils.PlatformUtil; import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProcessTerminator; +import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProgress; import org.sleuthkit.autopsy.ingest.IngestJobContext; import org.sleuthkit.datamodel.*; @@ -71,6 +73,16 @@ class ExtractIE extends Extract { private static final SimpleDateFormat dateFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); private Content dataSource; private IngestJobContext context; + + @Messages({ + "Progress_Message_IE_History=IE History", + "Progress_Message_IE_Bookmarks=IE Bookmarks", + "Progress_Message_IE_Cookies=IE Cookies", + "Progress_Message_IE_Downloads=IE Downloads", + "Progress_Message_IE_FormHistory=IE Form History", + "Progress_Message_IE_AutoFill=IE Auto Fill", + "Progress_Message_IE_Logins=IE Logins", + }) ExtractIE() throws NoCurrentCaseException { moduleName = NbBundle.getMessage(ExtractIE.class, "ExtractIE.moduleName.text"); @@ -79,12 +91,18 @@ class ExtractIE extends Extract { } @Override - public void process(Content dataSource, IngestJobContext context) { + public void process(Content dataSource, IngestJobContext context, DataSourceIngestModuleProgress progressBar) { this.dataSource = dataSource; this.context = context; dataFound = false; + + progressBar.progress(Bundle.Progress_Message_IE_Bookmarks()); this.getBookmark(); + + progressBar.progress(Bundle.Progress_Message_IE_Cookies()); this.getCookie(); + + progressBar.progress(Bundle.Progress_Message_IE_History()); this.getHistory(); } diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractOs.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractOs.java index 04976cb95b..9a838d2adb 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractOs.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractOs.java @@ -28,6 +28,7 @@ import org.apache.commons.io.FilenameUtils; import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.casemodule.services.FileManager; import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProgress; import org.sleuthkit.autopsy.ingest.IngestJobContext; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.BlackboardArtifact; @@ -39,7 +40,8 @@ import org.sleuthkit.datamodel.TskCoreException; * Create OS INFO artifacts for the Operating Systems believed to be present on * the data source. */ -@Messages({"ExtractOs.parentModuleName=Recent Activity"}) +@Messages({"ExtractOs.parentModuleName=Recent Activity", + "ExtractOS_progressMessage=Checking for OS"}) class ExtractOs extends Extract { private static final Logger logger = Logger.getLogger(ExtractOs.class.getName()); @@ -64,9 +66,10 @@ class ExtractOs extends Extract { private Content dataSource; @Override - void process(Content dataSource, IngestJobContext context) { + void process(Content dataSource, IngestJobContext context, DataSourceIngestModuleProgress progressBar) { this.dataSource = dataSource; try { + progressBar.progress(Bundle.ExtractOS_progressMessage()); for (OS_TYPE value : OS_TYPE.values()) { checkForOSFiles(value); } diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java index 68885bcc52..77338655a5 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java @@ -52,6 +52,7 @@ import org.xml.sax.SAXException; import java.nio.file.Path; import static java.util.TimeZone.getTimeZone; import org.openide.util.Lookup; +import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProgress; import org.sleuthkit.autopsy.ingest.IngestModule.IngestModuleException; import org.sleuthkit.autopsy.ingest.IngestServices; import org.sleuthkit.autopsy.ingest.ModuleDataEvent; @@ -66,7 +67,8 @@ import org.sleuthkit.datamodel.ReadContentInputStream.ReadContentInputStreamExce */ @NbBundle.Messages({ "RegRipperNotFound=Autopsy RegRipper executable not found.", - "RegRipperFullNotFound=Full version RegRipper executable not found." + "RegRipperFullNotFound=Full version RegRipper executable not found.", + "Progress_Message_Analyze_Registry=Analyzing Registry Files" }) class ExtractRegistry extends Extract { @@ -969,9 +971,11 @@ class ExtractRegistry extends Extract { } @Override - public void process(Content dataSource, IngestJobContext context) { + public void process(Content dataSource, IngestJobContext context, DataSourceIngestModuleProgress progressBar) { this.dataSource = dataSource; this.context = context; + + progressBar.progress(Bundle.Progress_Message_Analyze_Registry()); analyzeRegistryFiles(); } diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractSafari.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractSafari.java index afb52654b2..3f98bcdf91 100755 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractSafari.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractSafari.java @@ -41,6 +41,7 @@ import org.sleuthkit.autopsy.casemodule.services.FileManager; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.NetworkUtils; import org.sleuthkit.autopsy.datamodel.ContentUtils; +import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProgress; import org.sleuthkit.autopsy.ingest.IngestJobContext; import org.sleuthkit.autopsy.ingest.IngestServices; import org.sleuthkit.autopsy.ingest.ModuleDataEvent; @@ -89,6 +90,10 @@ final class ExtractSafari extends Extract { "ExtractSafari_Error_Getting_History=An error occurred while processing Safari history files.", "ExtractSafari_Error_Parsing_Bookmark=An error occured while processing Safari Bookmark files", "ExtractSafari_Error_Parsing_Cookies=An error occured while processing Safari Cookies files", + "Progress_Message_Safari_History=Safari History", + "Progress_Message_Safari_Bookmarks=Safari Bookmarks", + "Progress_Message_Safari_Cookies=Safari Cookies", + "Progress_Message_Safari_Downloads=Safari Downloads", }) /** @@ -105,9 +110,10 @@ final class ExtractSafari extends Extract { } @Override - void process(Content dataSource, IngestJobContext context) { + void process(Content dataSource, IngestJobContext context, DataSourceIngestModuleProgress progressBar) { setFoundData(false); - + + progressBar.progress(Bundle.Progress_Message_Safari_Cookies()); try { processHistoryDB(dataSource, context); @@ -116,6 +122,7 @@ final class ExtractSafari extends Extract { LOG.log(Level.SEVERE, "Exception thrown while processing history file: {0}", ex); //NON-NLS } + progressBar.progress(Bundle.Progress_Message_Safari_Bookmarks()); try { processBookmarkPList(dataSource, context); } catch (IOException | TskCoreException | SAXException | PropertyListFormatException | ParseException | ParserConfigurationException ex) { @@ -123,13 +130,15 @@ final class ExtractSafari extends Extract { LOG.log(Level.SEVERE, "Exception thrown while parsing Safari Bookmarks file: {0}", ex); //NON-NLS } - try { + progressBar.progress(Bundle.Progress_Message_Safari_Downloads()); + try { processDownloadsPList(dataSource, context); } catch (IOException | TskCoreException | SAXException | PropertyListFormatException | ParseException | ParserConfigurationException ex) { this.addErrorMessage(Bundle.ExtractSafari_Error_Parsing_Bookmark()); LOG.log(Level.SEVERE, "Exception thrown while parsing Safari Download.plist file: {0}", ex); //NON-NLS } + progressBar.progress(Bundle.Progress_Message_Safari_Cookies()); try { processBinaryCookieFile(dataSource, context); } catch (IOException | TskCoreException ex) { diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Firefox.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Firefox.java index 043e0bf0ff..129d86d10d 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Firefox.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Firefox.java @@ -44,12 +44,14 @@ import java.util.Set; import java.util.logging.Level; import org.openide.util.NbBundle; +import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.autopsy.casemodule.services.FileManager; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.NetworkUtils; import org.sleuthkit.autopsy.datamodel.ContentUtils; +import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProgress; import org.sleuthkit.autopsy.ingest.IngestJobContext; import org.sleuthkit.autopsy.ingest.IngestServices; import org.sleuthkit.autopsy.ingest.ModuleDataEvent; @@ -63,6 +65,15 @@ import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.ReadContentInputStream.ReadContentInputStreamException; import org.sleuthkit.datamodel.TskCoreException; +@Messages({ + "Progress_Message_Firefox_History=Firefox History", + "Progress_Message_Firefox_Bookmarks=Firefox Bookmarks", + "Progress_Message_Firefox_Cookies=Firefox Cookies", + "Progress_Message_Firefox_Downloads=Firefox Downloads", + "Progress_Message_Firefox_FormHistory=Firefox Form History", + "Progress_Message_Firefox_AutoFill=Firefox Auto Fill" +}) + /** * Firefox recent activity extraction */ @@ -95,15 +106,27 @@ class Firefox extends Extract { } @Override - public void process(Content dataSource, IngestJobContext context) { + public void process(Content dataSource, IngestJobContext context, DataSourceIngestModuleProgress progressBar) { this.dataSource = dataSource; this.context = context; dataFound = false; + + progressBar.progress(Bundle.Progress_Message_Firefox_History()); this.getHistory(); + + progressBar.progress(Bundle.Progress_Message_Firefox_Bookmarks()); this.getBookmark(); + + progressBar.progress(Bundle.Progress_Message_Firefox_Downloads()); this.getDownload(); + + progressBar.progress(Bundle.Progress_Message_Firefox_Cookies()); this.getCookie(); + + progressBar.progress(Bundle.Progress_Message_Firefox_FormHistory()); this.getFormsHistory(); + + progressBar.progress(Bundle.Progress_Message_Firefox_AutoFill()); this.getAutofillProfiles(); } diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RAImageIngestModule.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RAImageIngestModule.java index 0a42d7a031..b81a512ecb 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RAImageIngestModule.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RAImageIngestModule.java @@ -120,7 +120,7 @@ public final class RAImageIngestModule implements DataSourceIngestModule { progressBar.progress(extracter.getName(), i); try { - extracter.process(dataSource, context); + extracter.process(dataSource, context, progressBar); } catch (Exception ex) { logger.log(Level.SEVERE, "Exception occurred in " + extracter.getName(), ex); //NON-NLS subCompleted.append(NbBundle.getMessage(this.getClass(), "RAImageIngestModule.process.errModFailed", diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RecentDocumentsByLnk.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RecentDocumentsByLnk.java index bb200551fb..160f88f9b7 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RecentDocumentsByLnk.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RecentDocumentsByLnk.java @@ -29,6 +29,7 @@ import java.util.logging.Level; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.coreutils.Logger; import java.util.Collection; +import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.coreutils.JLNK; import org.sleuthkit.autopsy.coreutils.JLnkParser; import org.sleuthkit.autopsy.coreutils.JLnkParserException; @@ -53,6 +54,10 @@ class RecentDocumentsByLnk extends Extract { private IngestServices services = IngestServices.getInstance(); private Content dataSource; private IngestJobContext context; + + @Messages({ + "Progress_Message_Extract_Resent_Docs=Recent Documents", + }) /** * Find the documents that Windows stores about recent documents and make @@ -125,10 +130,12 @@ class RecentDocumentsByLnk extends Extract { } @Override - public void process(Content dataSource, IngestJobContext context) { + public void process(Content dataSource, IngestJobContext context, DataSourceIngestModuleProgress progressBar) { this.dataSource = dataSource; this.context = context; dataFound = false; + + progressBar.progress(Bundle.Progress_Message_Extract_Resent_Docs()); this.getRecentDocuments(); } } diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java index 4c8999ec47..5aff1a4a0b 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java @@ -33,6 +33,7 @@ import org.openide.util.NbBundle; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.PlatformUtil; import org.sleuthkit.autopsy.coreutils.XMLUtil; +import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProgress; import org.sleuthkit.autopsy.ingest.IngestJobContext; import org.sleuthkit.autopsy.ingest.IngestModule.IngestModuleException; import org.sleuthkit.autopsy.ingest.IngestServices; @@ -62,7 +63,8 @@ import org.xml.sax.SAXException; "cannotBuildXmlParser=Unable to build XML parser: ", "cannotLoadSEUQA=Unable to load Search Engine URL Query Analyzer settings file, SEUQAMappings.xml: ", "cannotParseXml=Unable to parse XML file: ", - "# {0} - file name", "SearchEngineURLQueryAnalyzer.init.exception.msg=Unable to find {0}." + "# {0} - file name", "SearchEngineURLQueryAnalyzer.init.exception.msg=Unable to find {0}.", + "Progress_Message_Find_Search_Query=Find Search Queries" }) class SearchEngineURLQueryAnalyzer extends Extract { @@ -396,9 +398,11 @@ class SearchEngineURLQueryAnalyzer extends Extract { } @Override - public void process(Content dataSource, IngestJobContext context) { + public void process(Content dataSource, IngestJobContext context, DataSourceIngestModuleProgress progressBar) { this.dataSource = dataSource; this.context = context; + + progressBar.progress(Bundle.Progress_Message_Find_Search_Query()); this.findSearchQueries(); logger.log(Level.INFO, "Search Engine stats: \n{0}", getTotals()); //NON-NLS }