From 0dd1a8a521eef984ca73e43e83ba9defe4b751c6 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Wed, 27 Feb 2019 13:43:34 -0500 Subject: [PATCH 01/15] 4626 make constants static, update copyright date --- .../datamodel/PostgresEamDbSettings.java | 28 +++++++++---------- .../datamodel/SqliteEamDbSettings.java | 28 +++++++++---------- 2 files changed, 28 insertions(+), 28 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDbSettings.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDbSettings.java index c72cc5f58f..71f23f4317 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDbSettings.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDbSettings.java @@ -1,7 +1,7 @@ /* * Central Repository * - * Copyright 2015-2017 Basis Technology Corp. + * Copyright 2015-2019 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -43,16 +43,16 @@ import static org.sleuthkit.autopsy.centralrepository.datamodel.AbstractSqlEamDb public final class PostgresEamDbSettings { private final static Logger LOGGER = Logger.getLogger(PostgresEamDbSettings.class.getName()); - private final String DEFAULT_HOST = ""; // NON-NLS - private final int DEFAULT_PORT = 5432; - private final String DEFAULT_DBNAME = "central_repository"; // NON-NLS - private final String DEFAULT_USERNAME = ""; - private final String DEFAULT_PASSWORD = ""; - private final String VALIDATION_QUERY = "SELECT version()"; // NON-NLS - private final String JDBC_BASE_URI = "jdbc:postgresql://"; // NON-NLS - private final String JDBC_DRIVER = "org.postgresql.Driver"; // NON-NLS - private final String DB_NAMES_REGEX = "[a-z][a-z0-9_]*"; // only lower case - private final String DB_USER_NAMES_REGEX = "[a-zA-Z]\\w*"; + private final static String DEFAULT_HOST = ""; // NON-NLS + private final static int DEFAULT_PORT = 5432; + private final static String DEFAULT_DBNAME = "central_repository"; // NON-NLS + private final static String DEFAULT_USERNAME = ""; + private final static String DEFAULT_PASSWORD = ""; + private final static String VALIDATION_QUERY = "SELECT version()"; // NON-NLS + private final static String JDBC_BASE_URI = "jdbc:postgresql://"; // NON-NLS + private final static String JDBC_DRIVER = "org.postgresql.Driver"; // NON-NLS + private final static String DB_NAMES_REGEX = "[a-z][a-z0-9_]*"; // only lower case + private final static String DB_USER_NAMES_REGEX = "[a-zA-Z]\\w*"; private String host; private int port; private String dbName; @@ -425,7 +425,7 @@ public final class PostgresEamDbSettings { stmt.execute(createDataSourcesTable.toString()); stmt.execute(dataSourceIdx1); stmt.execute(dataSourceIdx2); - + stmt.execute(createReferenceSetsTable.toString()); stmt.execute(referenceSetsIdx1); @@ -561,8 +561,8 @@ public final class PostgresEamDbSettings { * instance table. %s will exist in the template where the name of the new * table will be addedd. * - * @return a String which is a template for adding an index to the file_obj_id - * column of a _instances table + * @return a String which is a template for adding an index to the + * file_obj_id column of a _instances table */ static String getAddObjectIdIndexTemplate() { // Each "%s" will be replaced with the relevant TYPE_instances table name. diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDbSettings.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDbSettings.java index 4baa423a04..784c93df33 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDbSettings.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDbSettings.java @@ -1,7 +1,7 @@ /* * Central Repository * - * Copyright 2015-2017 Basis Technology Corp. + * Copyright 2015-2019 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -43,19 +43,19 @@ import static org.sleuthkit.autopsy.centralrepository.datamodel.AbstractSqlEamDb public final class SqliteEamDbSettings { private final static Logger LOGGER = Logger.getLogger(SqliteEamDbSettings.class.getName()); - private final String DEFAULT_DBNAME = "central_repository.db"; // NON-NLS - private final String DEFAULT_DBDIRECTORY = PlatformUtil.getUserDirectory() + File.separator + "central_repository"; // NON-NLS - private final String JDBC_DRIVER = "org.sqlite.JDBC"; // NON-NLS - private final String JDBC_BASE_URI = "jdbc:sqlite:"; // NON-NLS - private final String VALIDATION_QUERY = "SELECT count(*) from sqlite_master"; // NON-NLS - private static final String PRAGMA_SYNC_OFF = "PRAGMA synchronous = OFF"; - private static final String PRAGMA_SYNC_NORMAL = "PRAGMA synchronous = NORMAL"; - private static final String PRAGMA_JOURNAL_WAL = "PRAGMA journal_mode = WAL"; - private static final String PRAGMA_READ_UNCOMMITTED_TRUE = "PRAGMA read_uncommitted = True"; - private static final String PRAGMA_ENCODING_UTF8 = "PRAGMA encoding = 'UTF-8'"; - private static final String PRAGMA_PAGE_SIZE_4096 = "PRAGMA page_size = 4096"; - private static final String PRAGMA_FOREIGN_KEYS_ON = "PRAGMA foreign_keys = ON"; - private final String DB_NAMES_REGEX = "[a-z][a-z0-9_]*(\\.db)?"; + private final static String DEFAULT_DBNAME = "central_repository.db"; // NON-NLS + private final static String DEFAULT_DBDIRECTORY = PlatformUtil.getUserDirectory() + File.separator + "central_repository"; // NON-NLS + private final static String JDBC_DRIVER = "org.sqlite.JDBC"; // NON-NLS + private final static String JDBC_BASE_URI = "jdbc:sqlite:"; // NON-NLS + private final static String VALIDATION_QUERY = "SELECT count(*) from sqlite_master"; // NON-NLS + private final static String PRAGMA_SYNC_OFF = "PRAGMA synchronous = OFF"; + private final static String PRAGMA_SYNC_NORMAL = "PRAGMA synchronous = NORMAL"; + private final static String PRAGMA_JOURNAL_WAL = "PRAGMA journal_mode = WAL"; + private final static String PRAGMA_READ_UNCOMMITTED_TRUE = "PRAGMA read_uncommitted = True"; + private final static String PRAGMA_ENCODING_UTF8 = "PRAGMA encoding = 'UTF-8'"; + private final static String PRAGMA_PAGE_SIZE_4096 = "PRAGMA page_size = 4096"; + private final static String PRAGMA_FOREIGN_KEYS_ON = "PRAGMA foreign_keys = ON"; + private final static String DB_NAMES_REGEX = "[a-z][a-z0-9_]*(\\.db)?"; private String dbName; private String dbDirectory; private int bulkThreshold; From d800c28c1b328f9fc0589c020832c430e7d822cb Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Wed, 27 Feb 2019 13:46:32 -0500 Subject: [PATCH 02/15] 4626 change unique constraint on cr data sources table for newly created CRs --- .../centralrepository/datamodel/PostgresEamDbSettings.java | 2 +- .../centralrepository/datamodel/SqliteEamDbSettings.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDbSettings.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDbSettings.java index 71f23f4317..b7c84979b0 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDbSettings.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDbSettings.java @@ -350,7 +350,7 @@ public final class PostgresEamDbSettings { createDataSourcesTable.append("sha1 text DEFAULT NULL,"); createDataSourcesTable.append("sha256 text DEFAULT NULL,"); createDataSourcesTable.append("foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL,"); - createDataSourcesTable.append("CONSTRAINT datasource_unique UNIQUE (case_id, device_id, name)"); + createDataSourcesTable.append("CONSTRAINT datasource_unique UNIQUE (case_id, device_id, name, datasource_obj_id)"); createDataSourcesTable.append(")"); String dataSourceIdx1 = "CREATE INDEX IF NOT EXISTS data_sources_name ON data_sources (name)"; diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDbSettings.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDbSettings.java index 784c93df33..53e9966104 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDbSettings.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDbSettings.java @@ -293,7 +293,7 @@ public final class SqliteEamDbSettings { createDataSourcesTable.append("sha1 text DEFAULT NULL,"); createDataSourcesTable.append("sha256 text DEFAULT NULL,"); createDataSourcesTable.append("foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL,"); - createDataSourcesTable.append("CONSTRAINT datasource_unique UNIQUE (case_id, device_id, name)"); + createDataSourcesTable.append("CONSTRAINT datasource_unique UNIQUE (case_id, device_id, name, datasource_obj_id)"); createDataSourcesTable.append(")"); String dataSourceIdx1 = "CREATE INDEX IF NOT EXISTS data_sources_name ON data_sources (name)"; From deb9670dd89dc19e138411e6b2bc5a6cad4f2475 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Wed, 27 Feb 2019 15:25:56 -0500 Subject: [PATCH 03/15] 4626 add upgrade code for Central repository data_sources unique constraint change --- .../datamodel/AbstractSqlEamDb.java | 31 +++++-- .../datamodel/PostgresEamDbSettings.java | 82 +++++++++++-------- .../datamodel/SqliteEamDbSettings.java | 81 ++++++++++-------- 3 files changed, 119 insertions(+), 75 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java index bfcbcb6cb5..0e4dfcc624 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java @@ -61,7 +61,7 @@ abstract class AbstractSqlEamDb implements EamDb { static final String SCHEMA_MINOR_VERSION_KEY = "SCHEMA_MINOR_VERSION"; static final String CREATION_SCHEMA_MAJOR_VERSION_KEY = "CREATION_SCHEMA_MAJOR_VERSION"; static final String CREATION_SCHEMA_MINOR_VERSION_KEY = "CREATION_SCHEMA_MINOR_VERSION"; - static final CaseDbSchemaVersionNumber SOFTWARE_CR_DB_SCHEMA_VERSION = new CaseDbSchemaVersionNumber(1, 2); + static final CaseDbSchemaVersionNumber SOFTWARE_CR_DB_SCHEMA_VERSION = new CaseDbSchemaVersionNumber(1, 3); protected final List defaultCorrelationTypes; @@ -625,7 +625,7 @@ abstract class AbstractSqlEamDb implements EamDb { // This data source is already in the central repo return eamDataSource; } - + Connection conn = connect(); PreparedStatement preparedStatement = null; @@ -650,7 +650,7 @@ abstract class AbstractSqlEamDb implements EamDb { /* * If nothing was inserted, then return the data source that * exists in the Central Repository. - * + * * This is expected to occur with PostgreSQL Central Repository * databases. */ @@ -675,7 +675,7 @@ abstract class AbstractSqlEamDb implements EamDb { * If an exception was thrown causing us to not return a new data * source, attempt to get an existing data source with the same case * ID and data source object ID. - * + * * This exception block is expected to occur with SQLite Central * Repository databases. */ @@ -3582,7 +3582,28 @@ abstract class AbstractSqlEamDb implements EamDb { statement.execute("INSERT INTO db_info (name, value) VALUES ('" + AbstractSqlEamDb.CREATION_SCHEMA_MAJOR_VERSION_KEY + "','" + creationMajorVer + "')"); statement.execute("INSERT INTO db_info (name, value) VALUES ('" + AbstractSqlEamDb.CREATION_SCHEMA_MINOR_VERSION_KEY + "','" + creationMinorVer + "')"); } - + /* + * Update to 1.3 + */ + if (dbSchemaVersion.compareTo(new CaseDbSchemaVersionNumber(1, 3)) < 0) { + switch (selectedPlatform) { + case POSTGRESQL: + statement.execute("ALTER TABLE data_sources DROP CONSTRAINT datasource_unique"); + statement.execute("ALTER TABLE data_sources ADD CONSTRAINT datasource_unique UNIQUE (case_id, device_id, name, datasource_obj_id)"); + + break; + case SQLITE: + statement.execute("ALTER TABLE data_sources RENAME TO old_data_sources"); + statement.execute(SqliteEamDbSettings.getCreateDataSourcesTableStatement()); + statement.execute(SqliteEamDbSettings.getAddDataSourcesNameIndexStatement()); + statement.execute(SqliteEamDbSettings.getAddDataSourcesObjectIdIndexStatement()); + statement.execute("INSERT INTO data_sources SELECT * FROM old_data_sources"); + statement.execute("DROP TABLE old_data_sources"); + break; + default: + throw new EamDbException("Currently selected database platform \"" + selectedPlatform.name() + "\" can not be upgraded."); + } + } updateSchemaVersion(conn); conn.commit(); logger.log(Level.INFO, String.format("Central Repository schema updated to version %s", SOFTWARE_CR_DB_SCHEMA_VERSION)); diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDbSettings.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDbSettings.java index b7c84979b0..5e6f7e53ed 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDbSettings.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDbSettings.java @@ -339,23 +339,6 @@ public final class PostgresEamDbSettings { String casesIdx1 = "CREATE INDEX IF NOT EXISTS cases_org_id ON cases (org_id)"; String casesIdx2 = "CREATE INDEX IF NOT EXISTS cases_case_uid ON cases (case_uid)"; - StringBuilder createDataSourcesTable = new StringBuilder(); - createDataSourcesTable.append("CREATE TABLE IF NOT EXISTS data_sources ("); - createDataSourcesTable.append("id SERIAL PRIMARY KEY,"); - createDataSourcesTable.append("case_id integer NOT NULL,"); - createDataSourcesTable.append("device_id text NOT NULL,"); - createDataSourcesTable.append("name text NOT NULL,"); - createDataSourcesTable.append("datasource_obj_id BIGINT,"); - createDataSourcesTable.append("md5 text DEFAULT NULL,"); - createDataSourcesTable.append("sha1 text DEFAULT NULL,"); - createDataSourcesTable.append("sha256 text DEFAULT NULL,"); - createDataSourcesTable.append("foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL,"); - createDataSourcesTable.append("CONSTRAINT datasource_unique UNIQUE (case_id, device_id, name, datasource_obj_id)"); - createDataSourcesTable.append(")"); - - String dataSourceIdx1 = "CREATE INDEX IF NOT EXISTS data_sources_name ON data_sources (name)"; - String dataSourceIdx2 = "CREATE INDEX IF NOT EXISTS data_sources_object_id ON data_sources (datasource_obj_id)"; - StringBuilder createReferenceSetsTable = new StringBuilder(); createReferenceSetsTable.append("CREATE TABLE IF NOT EXISTS reference_sets ("); createReferenceSetsTable.append("id SERIAL PRIMARY KEY,"); @@ -422,9 +405,9 @@ public final class PostgresEamDbSettings { stmt.execute(casesIdx1); stmt.execute(casesIdx2); - stmt.execute(createDataSourcesTable.toString()); - stmt.execute(dataSourceIdx1); - stmt.execute(dataSourceIdx2); + stmt.execute(getCreateDataSourcesTableStatement()); + stmt.execute(getAddDataSourcesNameIndexStatement()); + stmt.execute(getAddDataSourcesObjectIdIndexStatement()); stmt.execute(createReferenceSetsTable.toString()); stmt.execute(referenceSetsIdx1); @@ -487,21 +470,50 @@ public final class PostgresEamDbSettings { */ static String getCreateArtifactInstancesTableTemplate() { // Each "%s" will be replaced with the relevant TYPE_instances table name. - StringBuilder createArtifactInstancesTableTemplate = new StringBuilder(); - createArtifactInstancesTableTemplate.append("CREATE TABLE IF NOT EXISTS %s ("); - createArtifactInstancesTableTemplate.append("id SERIAL PRIMARY KEY,"); - createArtifactInstancesTableTemplate.append("case_id integer NOT NULL,"); - createArtifactInstancesTableTemplate.append("data_source_id integer NOT NULL,"); - createArtifactInstancesTableTemplate.append("value text NOT NULL,"); - createArtifactInstancesTableTemplate.append("file_path text NOT NULL,"); - createArtifactInstancesTableTemplate.append("known_status integer NOT NULL,"); - createArtifactInstancesTableTemplate.append("comment text,"); - createArtifactInstancesTableTemplate.append("file_obj_id BIGINT,"); - createArtifactInstancesTableTemplate.append("CONSTRAINT %s_multi_unique_ UNIQUE (data_source_id, value, file_path),"); - createArtifactInstancesTableTemplate.append("foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL,"); - createArtifactInstancesTableTemplate.append("foreign key (data_source_id) references data_sources(id) ON UPDATE SET NULL ON DELETE SET NULL"); - createArtifactInstancesTableTemplate.append(")"); - return createArtifactInstancesTableTemplate.toString(); + return ("CREATE TABLE IF NOT EXISTS %s (id SERIAL PRIMARY KEY,case_id integer NOT NULL," + + "data_source_id integer NOT NULL,value text NOT NULL,file_path text NOT NULL," + + "known_status integer NOT NULL,comment text,file_obj_id BIGINT," + + "CONSTRAINT %s_multi_unique_ UNIQUE (data_source_id, value, file_path)," + + "foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL," + + "foreign key (data_source_id) references data_sources(id) ON UPDATE SET NULL ON DELETE SET NULL)"); + } + + /** + * Get the statement String for creating a new data_sources table in a + * Postgres central repository. + * + * @return a String which is a statement for cretating a new data_sources + * table + */ + static String getCreateDataSourcesTableStatement() { + return "CREATE TABLE IF NOT EXISTS data_sources " + + "(id SERIAL PRIMARY KEY,case_id integer NOT NULL,device_id text NOT NULL," + + "name text NOT NULL,datasource_obj_id BIGINT,md5 text DEFAULT NULL," + + "sha1 text DEFAULT NULL,sha256 text DEFAULT NULL," + + "foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL," + + "CONSTRAINT datasource_unique UNIQUE (case_id, device_id, name, datasource_obj_id))"; + } + + /** + * Get the statement for creating an index on the name column of the + * data_sources table. + * + * @return a String which is a statement for adding an index on the name + * column of the data_sources table. + */ + static String getAddDataSourcesNameIndexStatement() { + return "CREATE INDEX IF NOT EXISTS data_sources_name ON data_sources (name)"; + } + + /** + * Get the statement for creating an index on the data_sources_object_id + * column of the data_sources table. + * + * @return a String which is a statement for adding an index on the + * data_sources_object_id column of the data_sources table. + */ + static String getAddDataSourcesObjectIdIndexStatement() { + return "CREATE INDEX IF NOT EXISTS data_sources_object_id ON data_sources (datasource_obj_id)"; } /** diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDbSettings.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDbSettings.java index 53e9966104..ae8e419e99 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDbSettings.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDbSettings.java @@ -282,23 +282,6 @@ public final class SqliteEamDbSettings { String casesIdx1 = "CREATE INDEX IF NOT EXISTS cases_org_id ON cases (org_id)"; String casesIdx2 = "CREATE INDEX IF NOT EXISTS cases_case_uid ON cases (case_uid)"; - StringBuilder createDataSourcesTable = new StringBuilder(); - createDataSourcesTable.append("CREATE TABLE IF NOT EXISTS data_sources ("); - createDataSourcesTable.append("id integer primary key autoincrement NOT NULL,"); - createDataSourcesTable.append("case_id integer NOT NULL,"); - createDataSourcesTable.append("device_id text NOT NULL,"); - createDataSourcesTable.append("name text NOT NULL,"); - createDataSourcesTable.append("datasource_obj_id integer,"); - createDataSourcesTable.append("md5 text DEFAULT NULL,"); - createDataSourcesTable.append("sha1 text DEFAULT NULL,"); - createDataSourcesTable.append("sha256 text DEFAULT NULL,"); - createDataSourcesTable.append("foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL,"); - createDataSourcesTable.append("CONSTRAINT datasource_unique UNIQUE (case_id, device_id, name, datasource_obj_id)"); - createDataSourcesTable.append(")"); - - String dataSourceIdx1 = "CREATE INDEX IF NOT EXISTS data_sources_name ON data_sources (name)"; - String dataSourceIdx2 = "CREATE INDEX IF NOT EXISTS data_sources_object_id ON data_sources (datasource_obj_id)"; - StringBuilder createReferenceSetsTable = new StringBuilder(); createReferenceSetsTable.append("CREATE TABLE IF NOT EXISTS reference_sets ("); createReferenceSetsTable.append("id integer primary key autoincrement NOT NULL,"); @@ -371,9 +354,9 @@ public final class SqliteEamDbSettings { stmt.execute(casesIdx1); stmt.execute(casesIdx2); - stmt.execute(createDataSourcesTable.toString()); - stmt.execute(dataSourceIdx1); - stmt.execute(dataSourceIdx2); + stmt.execute(getCreateDataSourcesTableStatement()); + stmt.execute(getAddDataSourcesNameIndexStatement()); + stmt.execute(getAddDataSourcesObjectIdIndexStatement()); stmt.execute(createReferenceSetsTable.toString()); stmt.execute(referenceSetsIdx1); @@ -435,21 +418,49 @@ public final class SqliteEamDbSettings { */ static String getCreateArtifactInstancesTableTemplate() { // Each "%s" will be replaced with the relevant TYPE_instances table name. - StringBuilder createArtifactInstancesTableTemplate = new StringBuilder(); - createArtifactInstancesTableTemplate.append("CREATE TABLE IF NOT EXISTS %s ("); - createArtifactInstancesTableTemplate.append("id integer primary key autoincrement NOT NULL,"); - createArtifactInstancesTableTemplate.append("case_id integer NOT NULL,"); - createArtifactInstancesTableTemplate.append("data_source_id integer NOT NULL,"); - createArtifactInstancesTableTemplate.append("value text NOT NULL,"); - createArtifactInstancesTableTemplate.append("file_path text NOT NULL,"); - createArtifactInstancesTableTemplate.append("known_status integer NOT NULL,"); - createArtifactInstancesTableTemplate.append("comment text,"); - createArtifactInstancesTableTemplate.append("file_obj_id integer,"); - createArtifactInstancesTableTemplate.append("CONSTRAINT %s_multi_unique UNIQUE(data_source_id, value, file_path) ON CONFLICT IGNORE,"); - createArtifactInstancesTableTemplate.append("foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL,"); - createArtifactInstancesTableTemplate.append("foreign key (data_source_id) references data_sources(id) ON UPDATE SET NULL ON DELETE SET NULL"); - createArtifactInstancesTableTemplate.append(")"); - return createArtifactInstancesTableTemplate.toString(); + return "CREATE TABLE IF NOT EXISTS %s (id integer primary key autoincrement NOT NULL," + + "case_id integer NOT NULL,data_source_id integer NOT NULL,value text NOT NULL," + + "file_path text NOT NULL,known_status integer NOT NULL,comment text,file_obj_id integer," + + "CONSTRAINT %s_multi_unique UNIQUE(data_source_id, value, file_path) ON CONFLICT IGNORE," + + "foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL," + + "foreign key (data_source_id) references data_sources(id) ON UPDATE SET NULL ON DELETE SET NULL)"; + } + + /** + * Get the statement String for creating a new data_sources table in a + * Sqlite central repository. + * + * @return a String which is a statement for cretating a new data_sources + * table + */ + static String getCreateDataSourcesTableStatement() { + return "CREATE TABLE IF NOT EXISTS data_sources (id integer primary key autoincrement NOT NULL," + + "case_id integer NOT NULL,device_id text NOT NULL,name text NOT NULL,datasource_obj_id integer," + + "md5 text DEFAULT NULL,sha1 text DEFAULT NULL,sha256 text DEFAULT NULL," + + "foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL," + + "CONSTRAINT datasource_unique UNIQUE (case_id, device_id, name, datasource_obj_id))"; + } + + /** + * Get the statement for creating an index on the name column of the + * data_sources table. + * + * @return a String which is a statement for adding an index on the name + * column of the data_sources table. + */ + static String getAddDataSourcesNameIndexStatement() { + return "CREATE INDEX IF NOT EXISTS data_sources_name ON data_sources (name)"; + } + + /** + * Get the statement for creating an index on the data_sources_object_id + * column of the data_sources table. + * + * @return a String which is a statement for adding an index on the + * data_sources_object_id column of the data_sources table. + */ + static String getAddDataSourcesObjectIdIndexStatement() { + return "CREATE INDEX IF NOT EXISTS data_sources_object_id ON data_sources (datasource_obj_id)"; } /** From effeceee29206653188459b53a9d4719ea89e046 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Thu, 28 Feb 2019 10:29:40 -0500 Subject: [PATCH 04/15] 4626 allow connection pool used for upgrades to ignore foreign keys with sqlite --- .../datamodel/AbstractSqlEamDb.java | 129 +++++++++--------- .../datamodel/PostgresEamDb.java | 7 +- .../datamodel/SqliteEamDb.java | 39 +++--- 3 files changed, 93 insertions(+), 82 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java index 0e4dfcc624..aa0c29a9cf 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java @@ -108,7 +108,7 @@ abstract class AbstractSqlEamDb implements EamDb { /** * Setup and create a connection to the selected database implementation */ - protected abstract Connection connect() throws EamDbException; + protected abstract Connection connect(boolean foreignKeys) throws EamDbException; /** * Add a new name/value pair in the db_info table. @@ -120,7 +120,7 @@ abstract class AbstractSqlEamDb implements EamDb { */ @Override public void newDbInfo(String name, String value) throws EamDbException { - Connection conn = connect(); + Connection conn = connect(true); PreparedStatement preparedStatement = null; String sql = "INSERT INTO db_info (name, value) VALUES (?, ?) " @@ -141,7 +141,7 @@ abstract class AbstractSqlEamDb implements EamDb { @Override public void addDataSourceObjectId(int rowId, long dataSourceObjectId) throws EamDbException { - Connection conn = connect(); + Connection conn = connect(true); PreparedStatement preparedStatement = null; String sql = "UPDATE data_sources SET datasource_obj_id=? WHERE id=?"; try { @@ -168,7 +168,7 @@ abstract class AbstractSqlEamDb implements EamDb { */ @Override public String getDbInfo(String name) throws EamDbException { - Connection conn = connect(); + Connection conn = connect(true); PreparedStatement preparedStatement = null; ResultSet resultSet = null; @@ -213,7 +213,7 @@ abstract class AbstractSqlEamDb implements EamDb { */ @Override public void updateDbInfo(String name, String value) throws EamDbException { - Connection conn = connect(); + Connection conn = connect(true); PreparedStatement preparedStatement = null; String sql = "UPDATE db_info SET value=? WHERE name=?"; @@ -252,7 +252,7 @@ abstract class AbstractSqlEamDb implements EamDb { return cRCase; } - Connection conn = connect(); + Connection conn = connect(true); PreparedStatement preparedStatement = null; String sql = "INSERT INTO cases(case_uid, org_id, case_name, creation_date, case_number, " @@ -362,7 +362,7 @@ abstract class AbstractSqlEamDb implements EamDb { throw new EamDbException("Correlation case is null"); } - Connection conn = connect(); + Connection conn = connect(true); PreparedStatement preparedStatement = null; String sql = "UPDATE cases " @@ -447,7 +447,7 @@ abstract class AbstractSqlEamDb implements EamDb { * @return The retrieved case */ private CorrelationCase getCaseByUUIDFromCr(String caseUUID) throws EamDbException { - Connection conn = connect(); + Connection conn = connect(true); CorrelationCase eamCaseResult = null; PreparedStatement preparedStatement = null; @@ -508,7 +508,7 @@ abstract class AbstractSqlEamDb implements EamDb { * @return The retrieved case */ private CorrelationCase getCaseByIdFromCr(int caseId) throws EamDbException { - Connection conn = connect(); + Connection conn = connect(true); CorrelationCase eamCaseResult = null; PreparedStatement preparedStatement = null; @@ -548,7 +548,7 @@ abstract class AbstractSqlEamDb implements EamDb { */ @Override public List getCases() throws EamDbException { - Connection conn = connect(); + Connection conn = connect(true); List cases = new ArrayList<>(); CorrelationCase eamCaseResult; @@ -626,7 +626,7 @@ abstract class AbstractSqlEamDb implements EamDb { return eamDataSource; } - Connection conn = connect(); + Connection conn = connect(true); PreparedStatement preparedStatement = null; //The conflict clause exists in case multiple nodes are trying to add the data source because it did not exist at the same time @@ -733,7 +733,7 @@ abstract class AbstractSqlEamDb implements EamDb { * @throws EamDbException */ private CorrelationDataSource getDataSourceFromCr(int correlationCaseId, Long dataSourceObjectId) throws EamDbException { - Connection conn = connect(); + Connection conn = connect(true); CorrelationDataSource eamDataSourceResult = null; PreparedStatement preparedStatement = null; @@ -797,7 +797,7 @@ abstract class AbstractSqlEamDb implements EamDb { * @return The data source */ private CorrelationDataSource getDataSourceByIdFromCr(CorrelationCase correlationCase, int dataSourceId) throws EamDbException { - Connection conn = connect(); + Connection conn = connect(true); CorrelationDataSource eamDataSourceResult = null; PreparedStatement preparedStatement = null; @@ -834,7 +834,7 @@ abstract class AbstractSqlEamDb implements EamDb { */ @Override public List getDataSources() throws EamDbException { - Connection conn = connect(); + Connection conn = connect(true); List dataSources = new ArrayList<>(); CorrelationDataSource eamDataSourceResult; @@ -904,7 +904,7 @@ abstract class AbstractSqlEamDb implements EamDb { throw new EamDbException("Correlation data source is null"); } - Connection conn = connect(); + Connection conn = connect(true); PreparedStatement preparedStatement = null; String sql = "UPDATE data_sources " @@ -940,7 +940,7 @@ abstract class AbstractSqlEamDb implements EamDb { @Override public void updateDataSourceName(CorrelationDataSource eamDataSource, String newName) throws EamDbException { - Connection conn = connect(); + Connection conn = connect(true); PreparedStatement preparedStatement = null; @@ -983,7 +983,7 @@ abstract class AbstractSqlEamDb implements EamDb { public void addArtifactInstance(CorrelationAttributeInstance eamArtifact) throws EamDbException { checkAddArtifactInstanceNulls(eamArtifact); - Connection conn = connect(); + Connection conn = connect(true); PreparedStatement preparedStatement = null; @@ -1068,7 +1068,7 @@ abstract class AbstractSqlEamDb implements EamDb { String normalizedValue = CorrelationAttributeNormalizer.normalize(aType, value); - Connection conn = connect(); + Connection conn = connect(true); List artifactInstances = new ArrayList<>(); @@ -1133,7 +1133,7 @@ abstract class AbstractSqlEamDb implements EamDb { if (filePath == null) { throw new EamDbException("Correlation value is null"); } - Connection conn = connect(); + Connection conn = connect(true); List artifactInstances = new ArrayList<>(); @@ -1197,7 +1197,7 @@ abstract class AbstractSqlEamDb implements EamDb { public Long getCountArtifactInstancesByTypeValue(CorrelationAttributeInstance.Type aType, String value) throws EamDbException, CorrelationAttributeNormalizationException { String normalizedValue = CorrelationAttributeNormalizer.normalize(aType, value); - Connection conn = connect(); + Connection conn = connect(true); Long instanceCount = 0L; PreparedStatement preparedStatement = null; @@ -1251,7 +1251,7 @@ abstract class AbstractSqlEamDb implements EamDb { public Long getCountUniqueCaseDataSourceTuplesHavingTypeValue(CorrelationAttributeInstance.Type aType, String value) throws EamDbException, CorrelationAttributeNormalizationException { String normalizedValue = CorrelationAttributeNormalizer.normalize(aType, value); - Connection conn = connect(); + Connection conn = connect(true); Long instanceCount = 0L; PreparedStatement preparedStatement = null; @@ -1284,7 +1284,7 @@ abstract class AbstractSqlEamDb implements EamDb { @Override public Long getCountUniqueDataSources() throws EamDbException { - Connection conn = connect(); + Connection conn = connect(true); Long instanceCount = 0L; PreparedStatement preparedStatement = null; @@ -1321,7 +1321,7 @@ abstract class AbstractSqlEamDb implements EamDb { */ @Override public Long getCountArtifactInstancesByCaseDataSource(CorrelationDataSource correlationDataSource) throws EamDbException { - Connection conn = connect(); + Connection conn = connect(true); Long instanceCount = 0L; List artifactTypes = getDefinedCorrelationTypes(); @@ -1394,7 +1394,7 @@ abstract class AbstractSqlEamDb implements EamDb { public void commitAttributeInstancesBulk() throws EamDbException { List artifactTypes = getDefinedCorrelationTypes(); - Connection conn = connect(); + Connection conn = connect(true); PreparedStatement bulkPs = null; try { @@ -1501,7 +1501,7 @@ abstract class AbstractSqlEamDb implements EamDb { return; } - Connection conn = connect(); + Connection conn = connect(true); int counter = 0; PreparedStatement bulkPs = null; @@ -1589,7 +1589,7 @@ abstract class AbstractSqlEamDb implements EamDb { if (eamArtifact.getCorrelationDataSource() == null) { throw new EamDbException("Correlation data source is null"); } - Connection conn = connect(); + Connection conn = connect(true); PreparedStatement preparedQuery = null; String tableName = EamDbUtil.correlationTypeToInstanceTableName(eamArtifact.getCorrelationType()); String sqlUpdate @@ -1639,7 +1639,7 @@ abstract class AbstractSqlEamDb implements EamDb { throw new EamDbException("Correlation case is null"); } - Connection conn = connect(); + Connection conn = connect(true); PreparedStatement preparedStatement = null; ResultSet resultSet = null; @@ -1707,7 +1707,7 @@ abstract class AbstractSqlEamDb implements EamDb { throw new EamDbException("Correlation file path is null"); } - Connection conn = connect(); + Connection conn = connect(true); PreparedStatement preparedStatement = null; ResultSet resultSet = null; @@ -1776,7 +1776,7 @@ abstract class AbstractSqlEamDb implements EamDb { throw new EamDbException("Correlation data source is null"); } - Connection conn = connect(); + Connection conn = connect(true); PreparedStatement preparedUpdate = null; PreparedStatement preparedQuery = null; @@ -1858,7 +1858,7 @@ abstract class AbstractSqlEamDb implements EamDb { public List getArtifactInstancesKnownBad(CorrelationAttributeInstance.Type aType, String value) throws EamDbException, CorrelationAttributeNormalizationException { String normalizedValue = CorrelationAttributeNormalizer.normalize(aType, value); - Connection conn = connect(); + Connection conn = connect(true); List artifactInstances = new ArrayList<>(); @@ -1922,7 +1922,7 @@ abstract class AbstractSqlEamDb implements EamDb { throw new EamDbException("Correlation type is null"); } - Connection conn = connect(); + Connection conn = connect(true); List artifactInstances = new ArrayList<>(); @@ -1981,7 +1981,7 @@ abstract class AbstractSqlEamDb implements EamDb { String normalizedValue = CorrelationAttributeNormalizer.normalize(aType, value); - Connection conn = connect(); + Connection conn = connect(true); Long badInstances = 0L; PreparedStatement preparedStatement = null; @@ -2028,7 +2028,7 @@ abstract class AbstractSqlEamDb implements EamDb { String normalizedValue = CorrelationAttributeNormalizer.normalize(aType, value); - Connection conn = connect(); + Connection conn = connect(true); Collection caseNames = new LinkedHashSet<>(); @@ -2087,7 +2087,7 @@ abstract class AbstractSqlEamDb implements EamDb { * @throws EamDbException */ private void deleteReferenceSetEntry(int referenceSetID) throws EamDbException { - Connection conn = connect(); + Connection conn = connect(true); PreparedStatement preparedStatement = null; String sql = "DELETE FROM reference_sets WHERE id=?"; @@ -2113,7 +2113,7 @@ abstract class AbstractSqlEamDb implements EamDb { * @throws EamDbException */ private void deleteReferenceSetEntries(int referenceSetID) throws EamDbException { - Connection conn = connect(); + Connection conn = connect(true); PreparedStatement preparedStatement = null; String sql = "DELETE FROM %s WHERE reference_set_id=?"; @@ -2186,7 +2186,7 @@ abstract class AbstractSqlEamDb implements EamDb { String normalizeValued = CorrelationAttributeNormalizer.normalize(this.getCorrelationTypeById(correlationTypeID), value); - Connection conn = connect(); + Connection conn = connect(true); Long matchingInstances = 0L; PreparedStatement preparedStatement = null; @@ -2232,7 +2232,7 @@ abstract class AbstractSqlEamDb implements EamDb { return false; } - Connection conn = connect(); + Connection conn = connect(true); Long badInstances = 0L; PreparedStatement preparedStatement = null; @@ -2275,7 +2275,7 @@ abstract class AbstractSqlEamDb implements EamDb { throw new EamDbException("Callback interface is null"); } - Connection conn = connect(); + Connection conn = connect(true); PreparedStatement preparedStatement = null; ResultSet resultSet = null; String tableName = EamDbUtil.correlationTypeToInstanceTableName(type); @@ -2319,7 +2319,7 @@ abstract class AbstractSqlEamDb implements EamDb { throw new EamDbException("Where clause is null"); } - Connection conn = connect(); + Connection conn = connect(true); PreparedStatement preparedStatement = null; ResultSet resultSet = null; String tableName = EamDbUtil.correlationTypeToInstanceTableName(type); @@ -2350,7 +2350,7 @@ abstract class AbstractSqlEamDb implements EamDb { throw new EamDbException("EamOrganization already has an ID"); } - Connection conn = connect(); + Connection conn = connect(true); ResultSet generatedKeys = null; PreparedStatement preparedStatement = null; String sql = "INSERT INTO organizations(org_name, poc_name, poc_email, poc_phone) VALUES (?, ?, ?, ?) " @@ -2389,7 +2389,7 @@ abstract class AbstractSqlEamDb implements EamDb { */ @Override public List getOrganizations() throws EamDbException { - Connection conn = connect(); + Connection conn = connect(true); List orgs = new ArrayList<>(); PreparedStatement preparedStatement = null; @@ -2424,7 +2424,7 @@ abstract class AbstractSqlEamDb implements EamDb { */ @Override public EamOrganization getOrganizationByID(int orgID) throws EamDbException { - Connection conn = connect(); + Connection conn = connect(true); PreparedStatement preparedStatement = null; ResultSet resultSet = null; @@ -2492,7 +2492,7 @@ abstract class AbstractSqlEamDb implements EamDb { public void updateOrganization(EamOrganization updatedOrganization) throws EamDbException { testArgument(updatedOrganization); - Connection conn = connect(); + Connection conn = connect(true); PreparedStatement preparedStatement = null; String sql = "UPDATE organizations SET org_name = ?, poc_name = ?, poc_email = ?, poc_phone = ? WHERE id = ?"; try { @@ -2515,7 +2515,7 @@ abstract class AbstractSqlEamDb implements EamDb { public void deleteOrganization(EamOrganization organizationToDelete) throws EamDbException { testArgument(organizationToDelete); - Connection conn = connect(); + Connection conn = connect(true); PreparedStatement checkIfUsedStatement = null; ResultSet resultSet = null; String checkIfUsedSql = "SELECT (select count(*) FROM cases WHERE org_id=?) + (select count(*) FROM reference_sets WHERE org_id=?)"; @@ -2566,7 +2566,7 @@ abstract class AbstractSqlEamDb implements EamDb { throw new EamDbException("Type on the EamGlobalSet is null"); } - Connection conn = connect(); + Connection conn = connect(true); PreparedStatement preparedStatement1 = null; PreparedStatement preparedStatement2 = null; @@ -2618,7 +2618,7 @@ abstract class AbstractSqlEamDb implements EamDb { */ @Override public EamGlobalSet getReferenceSetByID(int referenceSetID) throws EamDbException { - Connection conn = connect(); + Connection conn = connect(true); PreparedStatement preparedStatement1 = null; ResultSet resultSet = null; @@ -2660,7 +2660,7 @@ abstract class AbstractSqlEamDb implements EamDb { } List results = new ArrayList<>(); - Connection conn = connect(); + Connection conn = connect(true); PreparedStatement preparedStatement1 = null; ResultSet resultSet = null; @@ -2701,7 +2701,7 @@ abstract class AbstractSqlEamDb implements EamDb { throw new EamDbException("Correlation type is null"); } - Connection conn = connect(); + Connection conn = connect(true); PreparedStatement preparedStatement = null; @@ -2737,7 +2737,7 @@ abstract class AbstractSqlEamDb implements EamDb { */ @Override public boolean referenceSetExists(String referenceSetName, String version) throws EamDbException { - Connection conn = connect(); + Connection conn = connect(true); PreparedStatement preparedStatement1 = null; ResultSet resultSet = null; @@ -2774,7 +2774,7 @@ abstract class AbstractSqlEamDb implements EamDb { throw new EamDbException("Null set of EamGlobalFileInstance"); } - Connection conn = connect(); + Connection conn = connect(true); PreparedStatement bulkPs = null; try { @@ -2827,7 +2827,7 @@ abstract class AbstractSqlEamDb implements EamDb { public List getReferenceInstancesByTypeValue(CorrelationAttributeInstance.Type aType, String aValue) throws EamDbException, CorrelationAttributeNormalizationException { String normalizeValued = CorrelationAttributeNormalizer.normalize(aType, aValue); - Connection conn = connect(); + Connection conn = connect(true); List globalFileInstances = new ArrayList<>(); PreparedStatement preparedStatement1 = null; @@ -2888,7 +2888,7 @@ abstract class AbstractSqlEamDb implements EamDb { * @throws EamDbException */ public int newCorrelationTypeNotKnownId(CorrelationAttributeInstance.Type newType) throws EamDbException { - Connection conn = connect(); + Connection conn = connect(true); PreparedStatement preparedStatement = null; PreparedStatement preparedStatementQuery = null; @@ -2941,7 +2941,7 @@ abstract class AbstractSqlEamDb implements EamDb { * @throws EamDbException */ private int newCorrelationTypeKnownId(CorrelationAttributeInstance.Type newType) throws EamDbException { - Connection conn = connect(); + Connection conn = connect(true); PreparedStatement preparedStatement = null; PreparedStatement preparedStatementQuery = null; @@ -2987,7 +2987,7 @@ abstract class AbstractSqlEamDb implements EamDb { @Override public List getDefinedCorrelationTypes() throws EamDbException { - Connection conn = connect(); + Connection conn = connect(true); List aTypes = new ArrayList<>(); PreparedStatement preparedStatement = null; @@ -3022,7 +3022,7 @@ abstract class AbstractSqlEamDb implements EamDb { */ @Override public List getEnabledCorrelationTypes() throws EamDbException { - Connection conn = connect(); + Connection conn = connect(true); List aTypes = new ArrayList<>(); PreparedStatement preparedStatement = null; @@ -3057,7 +3057,7 @@ abstract class AbstractSqlEamDb implements EamDb { */ @Override public List getSupportedCorrelationTypes() throws EamDbException { - Connection conn = connect(); + Connection conn = connect(true); List aTypes = new ArrayList<>(); PreparedStatement preparedStatement = null; @@ -3090,7 +3090,7 @@ abstract class AbstractSqlEamDb implements EamDb { */ @Override public void updateCorrelationType(CorrelationAttributeInstance.Type aType) throws EamDbException { - Connection conn = connect(); + Connection conn = connect(true); PreparedStatement preparedStatement = null; String sql = "UPDATE correlation_types SET display_name=?, db_table_name=?, supported=?, enabled=? WHERE id=?"; @@ -3144,7 +3144,7 @@ abstract class AbstractSqlEamDb implements EamDb { * @throws EamDbException */ private CorrelationAttributeInstance.Type getCorrelationTypeByIdFromCr(int typeId) throws EamDbException { - Connection conn = connect(); + Connection conn = connect(true); CorrelationAttributeInstance.Type aType; PreparedStatement preparedStatement = null; @@ -3338,12 +3338,13 @@ abstract class AbstractSqlEamDb implements EamDb { Statement statement = null; PreparedStatement preparedStatement = null; Connection conn = null; + EamDbPlatformEnum selectedPlatform = null; try { - conn = connect(); + conn = connect(false); conn.setAutoCommit(false); statement = conn.createStatement(); - + selectedPlatform = EamDbPlatformEnum.getSelectedPlatform(); int minorVersion = 0; String minorVersionStr = null; resultSet = statement.executeQuery("SELECT value FROM db_info WHERE name='" + AbstractSqlEamDb.SCHEMA_MINOR_VERSION_KEY + "'"); @@ -3397,8 +3398,6 @@ abstract class AbstractSqlEamDb implements EamDb { return; } - EamDbPlatformEnum selectedPlatform = EamDbPlatformEnum.getSelectedPlatform(); - /* * Update to 1.1 */ @@ -3590,10 +3589,12 @@ abstract class AbstractSqlEamDb implements EamDb { case POSTGRESQL: statement.execute("ALTER TABLE data_sources DROP CONSTRAINT datasource_unique"); statement.execute("ALTER TABLE data_sources ADD CONSTRAINT datasource_unique UNIQUE (case_id, device_id, name, datasource_obj_id)"); - + break; case SQLITE: - statement.execute("ALTER TABLE data_sources RENAME TO old_data_sources"); + statement.execute("DROP INDEX IF EXISTS data_sources_name"); + statement.execute("DROP INDEX IF EXISTS data_sources_object_id"); + statement.execute("ALTER TABLE data_sources RENAME TO old_data_sources"); statement.execute(SqliteEamDbSettings.getCreateDataSourcesTableStatement()); statement.execute(SqliteEamDbSettings.getAddDataSourcesNameIndexStatement()); statement.execute(SqliteEamDbSettings.getAddDataSourcesObjectIdIndexStatement()); diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDb.java index 769b49bfd3..f4aff71927 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDb.java @@ -104,7 +104,7 @@ final class PostgresEamDb extends AbstractSqlEamDb { @Override public void reset() throws EamDbException { - Connection conn = connect(); + Connection conn = connect(true); try { Statement dropContent = conn.createStatement(); @@ -164,12 +164,15 @@ final class PostgresEamDb extends AbstractSqlEamDb { /** * Lazily setup Singleton connection on first request. * + * @param foreignKeys -ignored arguement with postgres databases + * * @return A connection from the connection pool. * * @throws EamDbException */ @Override - protected Connection connect() throws EamDbException { + protected Connection connect(boolean foreignKeys) throws EamDbException { + //foreignKeys boolean is ignored for postgres synchronized (this) { if (!EamDb.isEnabled()) { throw new EamDbException("Central Repository module is not enabled"); // NON-NLS diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDb.java index a9d3b8b46d..020bd69b96 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDb.java @@ -114,7 +114,7 @@ final class SqliteEamDb extends AbstractSqlEamDb { try { acquireExclusiveLock(); - Connection conn = connect(); + Connection conn = connect(true); try { @@ -153,7 +153,7 @@ final class SqliteEamDb extends AbstractSqlEamDb { * Setup a connection pool for db connections. * */ - private void setupConnectionPool() throws EamDbException { + private void setupConnectionPool(boolean foreignKeysEnabled) throws EamDbException { if (dbSettings.dbFileExists() == false) { throw new EamDbException("Central repository database missing"); @@ -169,25 +169,31 @@ final class SqliteEamDb extends AbstractSqlEamDb { connectionPool.setMaxIdle(-1); connectionPool.setMaxWaitMillis(1000); connectionPool.setValidationQuery(dbSettings.getValidationQuery()); - connectionPool.setConnectionInitSqls(Arrays.asList("PRAGMA foreign_keys = ON")); + if (foreignKeysEnabled) { + connectionPool.setConnectionInitSqls(Arrays.asList("PRAGMA foreign_keys = ON")); + } else { + connectionPool.setConnectionInitSqls(Arrays.asList("PRAGMA foreign_keys = OFF")); + } } /** * Lazily setup Singleton connection on first request. * + * @param foreignKeys determines if foreign keys should be enforced during this connection for SQLite + * * @return A connection from the connection pool. * * @throws EamDbException */ @Override - protected Connection connect() throws EamDbException { + protected Connection connect(boolean foreignKeys) throws EamDbException { synchronized (this) { if (!EamDb.isEnabled()) { throw new EamDbException("Central Repository module is not enabled"); // NON-NLS } if (connectionPool == null) { - setupConnectionPool(); + setupConnectionPool(foreignKeys); } try { @@ -275,7 +281,7 @@ final class SqliteEamDb extends AbstractSqlEamDb { } @Override - public void addDataSourceObjectId(int rowId, long dataSourceObjectId) throws EamDbException{ + public void addDataSourceObjectId(int rowId, long dataSourceObjectId) throws EamDbException { try { acquireExclusiveLock(); super.addDataSourceObjectId(rowId, dataSourceObjectId); @@ -433,14 +439,14 @@ final class SqliteEamDb extends AbstractSqlEamDb { releaseSharedLock(); } } - + /** * Changes the name of a data source in the DB - * - * @param eamDataSource The data source - * @param newName The new name - * - * @throws EamDbException + * + * @param eamDataSource The data source + * @param newName The new name + * + * @throws EamDbException */ @Override public void updateDataSourceName(CorrelationDataSource eamDataSource, String newName) throws EamDbException { @@ -451,7 +457,7 @@ final class SqliteEamDb extends AbstractSqlEamDb { releaseExclusiveLock(); } } - + /** * Updates the MD5 hash value in an existing data source in the database. * @@ -466,7 +472,7 @@ final class SqliteEamDb extends AbstractSqlEamDb { releaseExclusiveLock(); } } - + /** * Updates the SHA-1 hash value in an existing data source in the database. * @@ -481,9 +487,10 @@ final class SqliteEamDb extends AbstractSqlEamDb { releaseExclusiveLock(); } } - + /** - * Updates the SHA-256 hash value in an existing data source in the database. + * Updates the SHA-256 hash value in an existing data source in the + * database. * * @param eamDataSource The data source to update */ From b7dce55e28bf2cfeaa20c8f2c1204bd3c7be3c36 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Thu, 28 Feb 2019 10:31:44 -0500 Subject: [PATCH 05/15] 4626 auto format changes --- .../autopsy/centralrepository/datamodel/AbstractSqlEamDb.java | 2 +- .../autopsy/centralrepository/datamodel/PostgresEamDb.java | 2 +- .../centralrepository/datamodel/PostgresEamDbSettings.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java index aa0c29a9cf..f4fe77202f 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java @@ -3594,7 +3594,7 @@ abstract class AbstractSqlEamDb implements EamDb { case SQLITE: statement.execute("DROP INDEX IF EXISTS data_sources_name"); statement.execute("DROP INDEX IF EXISTS data_sources_object_id"); - statement.execute("ALTER TABLE data_sources RENAME TO old_data_sources"); + statement.execute("ALTER TABLE data_sources RENAME TO old_data_sources"); statement.execute(SqliteEamDbSettings.getCreateDataSourcesTableStatement()); statement.execute(SqliteEamDbSettings.getAddDataSourcesNameIndexStatement()); statement.execute(SqliteEamDbSettings.getAddDataSourcesObjectIdIndexStatement()); diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDb.java index f4aff71927..4a13e1e6b6 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDb.java @@ -165,7 +165,7 @@ final class PostgresEamDb extends AbstractSqlEamDb { * Lazily setup Singleton connection on first request. * * @param foreignKeys -ignored arguement with postgres databases - * + * * @return A connection from the connection pool. * * @throws EamDbException diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDbSettings.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDbSettings.java index 5e6f7e53ed..ff46697ddb 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDbSettings.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDbSettings.java @@ -477,7 +477,7 @@ public final class PostgresEamDbSettings { + "foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL," + "foreign key (data_source_id) references data_sources(id) ON UPDATE SET NULL ON DELETE SET NULL)"); } - + /** * Get the statement String for creating a new data_sources table in a * Postgres central repository. From 7c0c8de4886857fbfbf7403aef7478489f7fe35f Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Thu, 28 Feb 2019 13:44:09 -0500 Subject: [PATCH 06/15] 4626-change unique constraint on new central repository tables to be as minimal as possible --- .../centralrepository/datamodel/AbstractSqlEamDb.java | 8 +++++++- .../datamodel/PostgresEamDbSettings.java | 2 +- .../centralrepository/datamodel/SqliteEamDbSettings.java | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java index f4fe77202f..fe09da90ac 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java @@ -3588,6 +3588,7 @@ abstract class AbstractSqlEamDb implements EamDb { switch (selectedPlatform) { case POSTGRESQL: statement.execute("ALTER TABLE data_sources DROP CONSTRAINT datasource_unique"); + //unique constraint for upgraded data_sources table is purposefully different than new data_sources table statement.execute("ALTER TABLE data_sources ADD CONSTRAINT datasource_unique UNIQUE (case_id, device_id, name, datasource_obj_id)"); break; @@ -3595,7 +3596,12 @@ abstract class AbstractSqlEamDb implements EamDb { statement.execute("DROP INDEX IF EXISTS data_sources_name"); statement.execute("DROP INDEX IF EXISTS data_sources_object_id"); statement.execute("ALTER TABLE data_sources RENAME TO old_data_sources"); - statement.execute(SqliteEamDbSettings.getCreateDataSourcesTableStatement()); + //unique constraint for upgraded data_sources table is purposefully different than new data_sources table + statement.execute("CREATE TABLE IF NOT EXISTS data_sources (id integer primary key autoincrement NOT NULL," + + "case_id integer NOT NULL,device_id text NOT NULL,name text NOT NULL,datasource_obj_id integer," + + "md5 text DEFAULT NULL,sha1 text DEFAULT NULL,sha256 text DEFAULT NULL," + + "foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL," + + "CONSTRAINT datasource_unique UNIQUE (case_id, device_id, name, datasource_obj_id))"); statement.execute(SqliteEamDbSettings.getAddDataSourcesNameIndexStatement()); statement.execute(SqliteEamDbSettings.getAddDataSourcesObjectIdIndexStatement()); statement.execute("INSERT INTO data_sources SELECT * FROM old_data_sources"); diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDbSettings.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDbSettings.java index ff46697ddb..a4ab17906b 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDbSettings.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDbSettings.java @@ -491,7 +491,7 @@ public final class PostgresEamDbSettings { + "name text NOT NULL,datasource_obj_id BIGINT,md5 text DEFAULT NULL," + "sha1 text DEFAULT NULL,sha256 text DEFAULT NULL," + "foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL," - + "CONSTRAINT datasource_unique UNIQUE (case_id, device_id, name, datasource_obj_id))"; + + "CONSTRAINT datasource_unique UNIQUE (case_id, datasource_obj_id))"; } /** diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDbSettings.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDbSettings.java index ae8e419e99..c76ab19783 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDbSettings.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDbSettings.java @@ -438,7 +438,7 @@ public final class SqliteEamDbSettings { + "case_id integer NOT NULL,device_id text NOT NULL,name text NOT NULL,datasource_obj_id integer," + "md5 text DEFAULT NULL,sha1 text DEFAULT NULL,sha256 text DEFAULT NULL," + "foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL," - + "CONSTRAINT datasource_unique UNIQUE (case_id, device_id, name, datasource_obj_id))"; + + "CONSTRAINT datasource_unique UNIQUE (case_id, datasource_obj_id))"; } /** From ec3903cdb6a18c812e323cc8e885a43b9e6ee05b Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Fri, 1 Mar 2019 11:51:52 -0500 Subject: [PATCH 07/15] 4626 make connect(bool foreignKeys) an additional method instead of a replacement method --- .../datamodel/AbstractSqlEamDb.java | 119 +++++++++--------- .../datamodel/PostgresEamDb.java | 15 ++- .../datamodel/SqliteEamDb.java | 16 ++- 3 files changed, 88 insertions(+), 62 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java index fe09da90ac..504ae8040d 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java @@ -110,6 +110,11 @@ abstract class AbstractSqlEamDb implements EamDb { */ protected abstract Connection connect(boolean foreignKeys) throws EamDbException; + /** + * Setup and create a connection to the selected database implementation + */ + protected abstract Connection connect() throws EamDbException; + /** * Add a new name/value pair in the db_info table. * @@ -120,7 +125,7 @@ abstract class AbstractSqlEamDb implements EamDb { */ @Override public void newDbInfo(String name, String value) throws EamDbException { - Connection conn = connect(true); + Connection conn = connect(); PreparedStatement preparedStatement = null; String sql = "INSERT INTO db_info (name, value) VALUES (?, ?) " @@ -141,7 +146,7 @@ abstract class AbstractSqlEamDb implements EamDb { @Override public void addDataSourceObjectId(int rowId, long dataSourceObjectId) throws EamDbException { - Connection conn = connect(true); + Connection conn = connect(); PreparedStatement preparedStatement = null; String sql = "UPDATE data_sources SET datasource_obj_id=? WHERE id=?"; try { @@ -168,7 +173,7 @@ abstract class AbstractSqlEamDb implements EamDb { */ @Override public String getDbInfo(String name) throws EamDbException { - Connection conn = connect(true); + Connection conn = connect(); PreparedStatement preparedStatement = null; ResultSet resultSet = null; @@ -213,7 +218,7 @@ abstract class AbstractSqlEamDb implements EamDb { */ @Override public void updateDbInfo(String name, String value) throws EamDbException { - Connection conn = connect(true); + Connection conn = connect(); PreparedStatement preparedStatement = null; String sql = "UPDATE db_info SET value=? WHERE name=?"; @@ -252,7 +257,7 @@ abstract class AbstractSqlEamDb implements EamDb { return cRCase; } - Connection conn = connect(true); + Connection conn = connect(); PreparedStatement preparedStatement = null; String sql = "INSERT INTO cases(case_uid, org_id, case_name, creation_date, case_number, " @@ -362,7 +367,7 @@ abstract class AbstractSqlEamDb implements EamDb { throw new EamDbException("Correlation case is null"); } - Connection conn = connect(true); + Connection conn = connect(); PreparedStatement preparedStatement = null; String sql = "UPDATE cases " @@ -447,7 +452,7 @@ abstract class AbstractSqlEamDb implements EamDb { * @return The retrieved case */ private CorrelationCase getCaseByUUIDFromCr(String caseUUID) throws EamDbException { - Connection conn = connect(true); + Connection conn = connect(); CorrelationCase eamCaseResult = null; PreparedStatement preparedStatement = null; @@ -508,7 +513,7 @@ abstract class AbstractSqlEamDb implements EamDb { * @return The retrieved case */ private CorrelationCase getCaseByIdFromCr(int caseId) throws EamDbException { - Connection conn = connect(true); + Connection conn = connect(); CorrelationCase eamCaseResult = null; PreparedStatement preparedStatement = null; @@ -548,7 +553,7 @@ abstract class AbstractSqlEamDb implements EamDb { */ @Override public List getCases() throws EamDbException { - Connection conn = connect(true); + Connection conn = connect(); List cases = new ArrayList<>(); CorrelationCase eamCaseResult; @@ -626,7 +631,7 @@ abstract class AbstractSqlEamDb implements EamDb { return eamDataSource; } - Connection conn = connect(true); + Connection conn = connect(); PreparedStatement preparedStatement = null; //The conflict clause exists in case multiple nodes are trying to add the data source because it did not exist at the same time @@ -733,7 +738,7 @@ abstract class AbstractSqlEamDb implements EamDb { * @throws EamDbException */ private CorrelationDataSource getDataSourceFromCr(int correlationCaseId, Long dataSourceObjectId) throws EamDbException { - Connection conn = connect(true); + Connection conn = connect(); CorrelationDataSource eamDataSourceResult = null; PreparedStatement preparedStatement = null; @@ -797,7 +802,7 @@ abstract class AbstractSqlEamDb implements EamDb { * @return The data source */ private CorrelationDataSource getDataSourceByIdFromCr(CorrelationCase correlationCase, int dataSourceId) throws EamDbException { - Connection conn = connect(true); + Connection conn = connect(); CorrelationDataSource eamDataSourceResult = null; PreparedStatement preparedStatement = null; @@ -834,7 +839,7 @@ abstract class AbstractSqlEamDb implements EamDb { */ @Override public List getDataSources() throws EamDbException { - Connection conn = connect(true); + Connection conn = connect(); List dataSources = new ArrayList<>(); CorrelationDataSource eamDataSourceResult; @@ -904,7 +909,7 @@ abstract class AbstractSqlEamDb implements EamDb { throw new EamDbException("Correlation data source is null"); } - Connection conn = connect(true); + Connection conn = connect(); PreparedStatement preparedStatement = null; String sql = "UPDATE data_sources " @@ -940,7 +945,7 @@ abstract class AbstractSqlEamDb implements EamDb { @Override public void updateDataSourceName(CorrelationDataSource eamDataSource, String newName) throws EamDbException { - Connection conn = connect(true); + Connection conn = connect(); PreparedStatement preparedStatement = null; @@ -983,7 +988,7 @@ abstract class AbstractSqlEamDb implements EamDb { public void addArtifactInstance(CorrelationAttributeInstance eamArtifact) throws EamDbException { checkAddArtifactInstanceNulls(eamArtifact); - Connection conn = connect(true); + Connection conn = connect(); PreparedStatement preparedStatement = null; @@ -1068,7 +1073,7 @@ abstract class AbstractSqlEamDb implements EamDb { String normalizedValue = CorrelationAttributeNormalizer.normalize(aType, value); - Connection conn = connect(true); + Connection conn = connect(); List artifactInstances = new ArrayList<>(); @@ -1133,7 +1138,7 @@ abstract class AbstractSqlEamDb implements EamDb { if (filePath == null) { throw new EamDbException("Correlation value is null"); } - Connection conn = connect(true); + Connection conn = connect(); List artifactInstances = new ArrayList<>(); @@ -1197,7 +1202,7 @@ abstract class AbstractSqlEamDb implements EamDb { public Long getCountArtifactInstancesByTypeValue(CorrelationAttributeInstance.Type aType, String value) throws EamDbException, CorrelationAttributeNormalizationException { String normalizedValue = CorrelationAttributeNormalizer.normalize(aType, value); - Connection conn = connect(true); + Connection conn = connect(); Long instanceCount = 0L; PreparedStatement preparedStatement = null; @@ -1251,7 +1256,7 @@ abstract class AbstractSqlEamDb implements EamDb { public Long getCountUniqueCaseDataSourceTuplesHavingTypeValue(CorrelationAttributeInstance.Type aType, String value) throws EamDbException, CorrelationAttributeNormalizationException { String normalizedValue = CorrelationAttributeNormalizer.normalize(aType, value); - Connection conn = connect(true); + Connection conn = connect(); Long instanceCount = 0L; PreparedStatement preparedStatement = null; @@ -1284,7 +1289,7 @@ abstract class AbstractSqlEamDb implements EamDb { @Override public Long getCountUniqueDataSources() throws EamDbException { - Connection conn = connect(true); + Connection conn = connect(); Long instanceCount = 0L; PreparedStatement preparedStatement = null; @@ -1321,7 +1326,7 @@ abstract class AbstractSqlEamDb implements EamDb { */ @Override public Long getCountArtifactInstancesByCaseDataSource(CorrelationDataSource correlationDataSource) throws EamDbException { - Connection conn = connect(true); + Connection conn = connect(); Long instanceCount = 0L; List artifactTypes = getDefinedCorrelationTypes(); @@ -1394,7 +1399,7 @@ abstract class AbstractSqlEamDb implements EamDb { public void commitAttributeInstancesBulk() throws EamDbException { List artifactTypes = getDefinedCorrelationTypes(); - Connection conn = connect(true); + Connection conn = connect(); PreparedStatement bulkPs = null; try { @@ -1501,7 +1506,7 @@ abstract class AbstractSqlEamDb implements EamDb { return; } - Connection conn = connect(true); + Connection conn = connect(); int counter = 0; PreparedStatement bulkPs = null; @@ -1589,7 +1594,7 @@ abstract class AbstractSqlEamDb implements EamDb { if (eamArtifact.getCorrelationDataSource() == null) { throw new EamDbException("Correlation data source is null"); } - Connection conn = connect(true); + Connection conn = connect(); PreparedStatement preparedQuery = null; String tableName = EamDbUtil.correlationTypeToInstanceTableName(eamArtifact.getCorrelationType()); String sqlUpdate @@ -1639,7 +1644,7 @@ abstract class AbstractSqlEamDb implements EamDb { throw new EamDbException("Correlation case is null"); } - Connection conn = connect(true); + Connection conn = connect(); PreparedStatement preparedStatement = null; ResultSet resultSet = null; @@ -1707,7 +1712,7 @@ abstract class AbstractSqlEamDb implements EamDb { throw new EamDbException("Correlation file path is null"); } - Connection conn = connect(true); + Connection conn = connect(); PreparedStatement preparedStatement = null; ResultSet resultSet = null; @@ -1776,7 +1781,7 @@ abstract class AbstractSqlEamDb implements EamDb { throw new EamDbException("Correlation data source is null"); } - Connection conn = connect(true); + Connection conn = connect(); PreparedStatement preparedUpdate = null; PreparedStatement preparedQuery = null; @@ -1858,7 +1863,7 @@ abstract class AbstractSqlEamDb implements EamDb { public List getArtifactInstancesKnownBad(CorrelationAttributeInstance.Type aType, String value) throws EamDbException, CorrelationAttributeNormalizationException { String normalizedValue = CorrelationAttributeNormalizer.normalize(aType, value); - Connection conn = connect(true); + Connection conn = connect(); List artifactInstances = new ArrayList<>(); @@ -1922,7 +1927,7 @@ abstract class AbstractSqlEamDb implements EamDb { throw new EamDbException("Correlation type is null"); } - Connection conn = connect(true); + Connection conn = connect(); List artifactInstances = new ArrayList<>(); @@ -1981,7 +1986,7 @@ abstract class AbstractSqlEamDb implements EamDb { String normalizedValue = CorrelationAttributeNormalizer.normalize(aType, value); - Connection conn = connect(true); + Connection conn = connect(); Long badInstances = 0L; PreparedStatement preparedStatement = null; @@ -2028,7 +2033,7 @@ abstract class AbstractSqlEamDb implements EamDb { String normalizedValue = CorrelationAttributeNormalizer.normalize(aType, value); - Connection conn = connect(true); + Connection conn = connect(); Collection caseNames = new LinkedHashSet<>(); @@ -2087,7 +2092,7 @@ abstract class AbstractSqlEamDb implements EamDb { * @throws EamDbException */ private void deleteReferenceSetEntry(int referenceSetID) throws EamDbException { - Connection conn = connect(true); + Connection conn = connect(); PreparedStatement preparedStatement = null; String sql = "DELETE FROM reference_sets WHERE id=?"; @@ -2113,7 +2118,7 @@ abstract class AbstractSqlEamDb implements EamDb { * @throws EamDbException */ private void deleteReferenceSetEntries(int referenceSetID) throws EamDbException { - Connection conn = connect(true); + Connection conn = connect(); PreparedStatement preparedStatement = null; String sql = "DELETE FROM %s WHERE reference_set_id=?"; @@ -2186,7 +2191,7 @@ abstract class AbstractSqlEamDb implements EamDb { String normalizeValued = CorrelationAttributeNormalizer.normalize(this.getCorrelationTypeById(correlationTypeID), value); - Connection conn = connect(true); + Connection conn = connect(); Long matchingInstances = 0L; PreparedStatement preparedStatement = null; @@ -2232,7 +2237,7 @@ abstract class AbstractSqlEamDb implements EamDb { return false; } - Connection conn = connect(true); + Connection conn = connect(); Long badInstances = 0L; PreparedStatement preparedStatement = null; @@ -2275,7 +2280,7 @@ abstract class AbstractSqlEamDb implements EamDb { throw new EamDbException("Callback interface is null"); } - Connection conn = connect(true); + Connection conn = connect(); PreparedStatement preparedStatement = null; ResultSet resultSet = null; String tableName = EamDbUtil.correlationTypeToInstanceTableName(type); @@ -2319,7 +2324,7 @@ abstract class AbstractSqlEamDb implements EamDb { throw new EamDbException("Where clause is null"); } - Connection conn = connect(true); + Connection conn = connect(); PreparedStatement preparedStatement = null; ResultSet resultSet = null; String tableName = EamDbUtil.correlationTypeToInstanceTableName(type); @@ -2350,7 +2355,7 @@ abstract class AbstractSqlEamDb implements EamDb { throw new EamDbException("EamOrganization already has an ID"); } - Connection conn = connect(true); + Connection conn = connect(); ResultSet generatedKeys = null; PreparedStatement preparedStatement = null; String sql = "INSERT INTO organizations(org_name, poc_name, poc_email, poc_phone) VALUES (?, ?, ?, ?) " @@ -2389,7 +2394,7 @@ abstract class AbstractSqlEamDb implements EamDb { */ @Override public List getOrganizations() throws EamDbException { - Connection conn = connect(true); + Connection conn = connect(); List orgs = new ArrayList<>(); PreparedStatement preparedStatement = null; @@ -2424,7 +2429,7 @@ abstract class AbstractSqlEamDb implements EamDb { */ @Override public EamOrganization getOrganizationByID(int orgID) throws EamDbException { - Connection conn = connect(true); + Connection conn = connect(); PreparedStatement preparedStatement = null; ResultSet resultSet = null; @@ -2492,7 +2497,7 @@ abstract class AbstractSqlEamDb implements EamDb { public void updateOrganization(EamOrganization updatedOrganization) throws EamDbException { testArgument(updatedOrganization); - Connection conn = connect(true); + Connection conn = connect(); PreparedStatement preparedStatement = null; String sql = "UPDATE organizations SET org_name = ?, poc_name = ?, poc_email = ?, poc_phone = ? WHERE id = ?"; try { @@ -2515,7 +2520,7 @@ abstract class AbstractSqlEamDb implements EamDb { public void deleteOrganization(EamOrganization organizationToDelete) throws EamDbException { testArgument(organizationToDelete); - Connection conn = connect(true); + Connection conn = connect(); PreparedStatement checkIfUsedStatement = null; ResultSet resultSet = null; String checkIfUsedSql = "SELECT (select count(*) FROM cases WHERE org_id=?) + (select count(*) FROM reference_sets WHERE org_id=?)"; @@ -2566,7 +2571,7 @@ abstract class AbstractSqlEamDb implements EamDb { throw new EamDbException("Type on the EamGlobalSet is null"); } - Connection conn = connect(true); + Connection conn = connect(); PreparedStatement preparedStatement1 = null; PreparedStatement preparedStatement2 = null; @@ -2618,7 +2623,7 @@ abstract class AbstractSqlEamDb implements EamDb { */ @Override public EamGlobalSet getReferenceSetByID(int referenceSetID) throws EamDbException { - Connection conn = connect(true); + Connection conn = connect(); PreparedStatement preparedStatement1 = null; ResultSet resultSet = null; @@ -2660,7 +2665,7 @@ abstract class AbstractSqlEamDb implements EamDb { } List results = new ArrayList<>(); - Connection conn = connect(true); + Connection conn = connect(); PreparedStatement preparedStatement1 = null; ResultSet resultSet = null; @@ -2701,7 +2706,7 @@ abstract class AbstractSqlEamDb implements EamDb { throw new EamDbException("Correlation type is null"); } - Connection conn = connect(true); + Connection conn = connect(); PreparedStatement preparedStatement = null; @@ -2737,7 +2742,7 @@ abstract class AbstractSqlEamDb implements EamDb { */ @Override public boolean referenceSetExists(String referenceSetName, String version) throws EamDbException { - Connection conn = connect(true); + Connection conn = connect(); PreparedStatement preparedStatement1 = null; ResultSet resultSet = null; @@ -2774,7 +2779,7 @@ abstract class AbstractSqlEamDb implements EamDb { throw new EamDbException("Null set of EamGlobalFileInstance"); } - Connection conn = connect(true); + Connection conn = connect(); PreparedStatement bulkPs = null; try { @@ -2827,7 +2832,7 @@ abstract class AbstractSqlEamDb implements EamDb { public List getReferenceInstancesByTypeValue(CorrelationAttributeInstance.Type aType, String aValue) throws EamDbException, CorrelationAttributeNormalizationException { String normalizeValued = CorrelationAttributeNormalizer.normalize(aType, aValue); - Connection conn = connect(true); + Connection conn = connect(); List globalFileInstances = new ArrayList<>(); PreparedStatement preparedStatement1 = null; @@ -2888,7 +2893,7 @@ abstract class AbstractSqlEamDb implements EamDb { * @throws EamDbException */ public int newCorrelationTypeNotKnownId(CorrelationAttributeInstance.Type newType) throws EamDbException { - Connection conn = connect(true); + Connection conn = connect(); PreparedStatement preparedStatement = null; PreparedStatement preparedStatementQuery = null; @@ -2941,7 +2946,7 @@ abstract class AbstractSqlEamDb implements EamDb { * @throws EamDbException */ private int newCorrelationTypeKnownId(CorrelationAttributeInstance.Type newType) throws EamDbException { - Connection conn = connect(true); + Connection conn = connect(); PreparedStatement preparedStatement = null; PreparedStatement preparedStatementQuery = null; @@ -2987,7 +2992,7 @@ abstract class AbstractSqlEamDb implements EamDb { @Override public List getDefinedCorrelationTypes() throws EamDbException { - Connection conn = connect(true); + Connection conn = connect(); List aTypes = new ArrayList<>(); PreparedStatement preparedStatement = null; @@ -3022,7 +3027,7 @@ abstract class AbstractSqlEamDb implements EamDb { */ @Override public List getEnabledCorrelationTypes() throws EamDbException { - Connection conn = connect(true); + Connection conn = connect(); List aTypes = new ArrayList<>(); PreparedStatement preparedStatement = null; @@ -3057,7 +3062,7 @@ abstract class AbstractSqlEamDb implements EamDb { */ @Override public List getSupportedCorrelationTypes() throws EamDbException { - Connection conn = connect(true); + Connection conn = connect(); List aTypes = new ArrayList<>(); PreparedStatement preparedStatement = null; @@ -3090,7 +3095,7 @@ abstract class AbstractSqlEamDb implements EamDb { */ @Override public void updateCorrelationType(CorrelationAttributeInstance.Type aType) throws EamDbException { - Connection conn = connect(true); + Connection conn = connect(); PreparedStatement preparedStatement = null; String sql = "UPDATE correlation_types SET display_name=?, db_table_name=?, supported=?, enabled=? WHERE id=?"; @@ -3144,7 +3149,7 @@ abstract class AbstractSqlEamDb implements EamDb { * @throws EamDbException */ private CorrelationAttributeInstance.Type getCorrelationTypeByIdFromCr(int typeId) throws EamDbException { - Connection conn = connect(true); + Connection conn = connect(); CorrelationAttributeInstance.Type aType; PreparedStatement preparedStatement = null; diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDb.java index 4a13e1e6b6..89382289f0 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDb.java @@ -104,7 +104,7 @@ final class PostgresEamDb extends AbstractSqlEamDb { @Override public void reset() throws EamDbException { - Connection conn = connect(true); + Connection conn = connect(); try { Statement dropContent = conn.createStatement(); @@ -173,6 +173,18 @@ final class PostgresEamDb extends AbstractSqlEamDb { @Override protected Connection connect(boolean foreignKeys) throws EamDbException { //foreignKeys boolean is ignored for postgres + return connect(); + } + + /** + * Lazily setup Singleton connection on first request. + * + * @return A connection from the connection pool. + * + * @throws EamDbException + */ + @Override + protected Connection connect() throws EamDbException { synchronized (this) { if (!EamDb.isEnabled()) { throw new EamDbException("Central Repository module is not enabled"); // NON-NLS @@ -182,7 +194,6 @@ final class PostgresEamDb extends AbstractSqlEamDb { setupConnectionPool(); } } - try { return connectionPool.getConnection(); } catch (SQLException ex) { diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDb.java index 020bd69b96..8a6c4467c1 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDb.java @@ -114,7 +114,7 @@ final class SqliteEamDb extends AbstractSqlEamDb { try { acquireExclusiveLock(); - Connection conn = connect(true); + Connection conn = connect(); try { @@ -191,11 +191,9 @@ final class SqliteEamDb extends AbstractSqlEamDb { if (!EamDb.isEnabled()) { throw new EamDbException("Central Repository module is not enabled"); // NON-NLS } - if (connectionPool == null) { setupConnectionPool(foreignKeys); } - try { return connectionPool.getConnection(); } catch (SQLException ex) { @@ -204,6 +202,18 @@ final class SqliteEamDb extends AbstractSqlEamDb { } } + /** + * Lazily setup Singleton connection on first request with foreign keys enforced. + * + * @return A connection from the connection pool. + * + * @throws EamDbException + */ + @Override + protected Connection connect() throws EamDbException { + return connect(true); + } + @Override protected String getConflictClause() { // For sqlite, our conflict clause is part of the table schema From 4065b9996723f73c74d3d51f2be024e61342c6ab Mon Sep 17 00:00:00 2001 From: Kelly Kelly Date: Mon, 4 Mar 2019 10:22:42 -0500 Subject: [PATCH 08/15] Cleaned up the download code to put it on hold, fixed code to make sure we clean up the temp files properly and cleaned up a few javadocs --- .../autopsy/recentactivity/ExtractEdge.java | 71 ++++++++++--------- 1 file changed, 39 insertions(+), 32 deletions(-) diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractEdge.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractEdge.java index 2264a83850..2484d41b4f 100755 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractEdge.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractEdge.java @@ -95,11 +95,13 @@ final class ExtractEdge extends Extract { private static final String EDGE_FAVORITE_FILE_NAME = "Favorites.csv"; //NON-NLS private static final String EDGE_OUTPUT_FILE_NAME = "Output.txt"; //NON-NLS private static final String EDGE_ERROR_FILE_NAME = "File.txt"; //NON-NLS + private static final String EDGE_WEBCACHE_FOLDER_NAME = "WebCache"; //NON-NLS + private static final String EDGE_SPARTAN_FOLDER_NAME = "MicrosoftEdge"; //NON-NLS private static final String ESE_TOOL_FOLDER = "ESEDatabaseView"; //NON-NLS private static final String EDGE_RESULT_FOLDER_NAME = "results"; //NON-NLS - private static final SimpleDateFormat DATE_FORMATTER = new SimpleDateFormat("MM/dd/yyyy hh:mm:ss a"); + private static final SimpleDateFormat DATE_FORMATTER = new SimpleDateFormat("MM/dd/yyyy hh:mm:ss a"); //NON-NLS @Messages({ "ExtractEdge_process_errMsg_unableFindESEViewer=Unable to find ESEDatabaseViewer", @@ -174,7 +176,7 @@ final class ExtractEdge extends Extract { try { this.processSpartanDbFile(esedumper, spartanFiles); } catch (IOException | TskCoreException ex) { - this.addErrorMessage(Bundle.ExtractEdge_process_errMsg_webcacheFail()); + this.addErrorMessage(Bundle.ExtractEdge_process_errMsg_spartanFail()); LOG.log(Level.SEVERE, "Error returned from processSpartanDbFile", ex); // NON-NLS } } @@ -228,13 +230,15 @@ final class ExtractEdge extends Extract { this.getCookies(webCacheFile, resultsDir); -// if (context.dataSourceIngestIsCancelled()) { -// return; -// } -// Putting downloads on hold -// this.getDownload(webCacheFile, resultsDir); } finally { tempWebCacheFile.delete(); + + // Emppty the result dir + File[] resultFiles = resultsDir.listFiles(); + for (File file : resultFiles) { + file.delete(); + } + resultsDir.delete(); } } @@ -283,6 +287,13 @@ final class ExtractEdge extends Extract { } finally { tempSpartanFile.delete(); + + // Empty the result dir + File[] resultFiles = resultsDir.listFiles(); + for (File file : resultFiles) { + file.delete(); + } + resultsDir.delete(); } } @@ -393,7 +404,7 @@ final class ExtractEdge extends Extract { } /** - * Queries for cookie files and adds artifacts + * Queries for cookie files and adds artifacts. * * @param origFile Original case file * @param resultDir Output directory of ESEDatabaseViewer @@ -445,7 +456,9 @@ final class ExtractEdge extends Extract { } /** - * Queries for download files and adds artifacts + * Queries for download files and adds artifacts. + * + * Leaving for future use. * * @param origFile Original case file * @param resultDir Output directory of ESEDatabaseViewer @@ -524,7 +537,7 @@ final class ExtractEdge extends Extract { private List fetchWebCacheDBFiles() throws TskCoreException { org.sleuthkit.autopsy.casemodule.services.FileManager fileManager = currentCase.getServices().getFileManager(); - return fileManager.findFiles(dataSource, EDGE_WEBCACHE_NAME, "WebCache"); //NON-NLS + return fileManager.findFiles(dataSource, EDGE_WEBCACHE_NAME, EDGE_WEBCACHE_FOLDER_NAME); //NON-NLS } /** @@ -536,7 +549,7 @@ final class ExtractEdge extends Extract { private List fetchSpartanDBFiles() throws TskCoreException { org.sleuthkit.autopsy.casemodule.services.FileManager fileManager = currentCase.getServices().getFileManager(); - return fileManager.findFiles(dataSource, EDGE_SPARTAN_NAME, "MicrosoftEdge"); //NON-NLS + return fileManager.findFiles(dataSource, EDGE_SPARTAN_NAME, EDGE_SPARTAN_FOLDER_NAME); //NON-NLS } /** @@ -622,7 +635,7 @@ final class ExtractEdge extends Extract { * @throws TskCoreException */ private BlackboardArtifact getCookieArtifact(AbstractFile origFile, List headers, String line) throws TskCoreException { - String[] lineSplit = line.split(","); + String[] lineSplit = line.split(","); // NON-NLS String accessTime = lineSplit[headers.indexOf(EDGE_HEAD_LASTMOD)].trim(); Long ftime = null; @@ -636,9 +649,10 @@ final class ExtractEdge extends Extract { String domain = lineSplit[headers.indexOf(EDGE_HEAD_RDOMAIN)].trim(); String name = hexToChar(lineSplit[headers.indexOf(EDGE_HEAD_NAME)].trim()); String value = hexToChar(lineSplit[headers.indexOf(EDGE_HEAD_VALUE)].trim()); + String url = flipDomain(domain); BlackboardArtifact bbart = origFile.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_COOKIE); - bbart.addAttributes(createCookieAttributes(null, ftime, name, value, this.getName(), flipDomain(domain))); + bbart.addAttributes(createCookieAttributes(url, ftime, name, value, this.getName(), NetworkUtils.extractDomain(url))); return bbart; } @@ -656,18 +670,12 @@ final class ExtractEdge extends Extract { * @throws TskCoreException */ private BlackboardArtifact getDownloadArtifact(AbstractFile origFile, List headers, String line) throws TskCoreException { - -// String[] lineSplit = line.split(","); -// -// String url = lineSplit[headers.indexOf(EDGE_HEAD_URL)]; -// -// String rheader = lineSplit[headers.indexOf(EDGE_HEAD_RESPONSEHEAD)]; -// -// String decodedheader = this.hexToASCII(rheader); -// BlackboardArtifact bbart = origFile.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_DOWNLOAD); -// bbart.addAttributes(createDownloadAttributes(decodedheader, "Test2", null, "microsoft.com", this.getName())); -// return bbart; - return null; + BlackboardArtifact bbart = null; + + String[] lineSplit = line.split(","); // NON-NLS + String rheader = lineSplit[headers.indexOf(EDGE_HEAD_RESPONSEHEAD)]; + + return bbart; } /** @@ -687,7 +695,7 @@ final class ExtractEdge extends Extract { String[] lineSplit = line.split(IGNORE_COMMA_IN_QUOTES_REGEX, -1); String url = lineSplit[headers.indexOf(EDGE_HEAD_URL)]; - String title = lineSplit[headers.indexOf(EDGE_HEAD_TITLE)].replace("\"", ""); + String title = lineSplit[headers.indexOf(EDGE_HEAD_TITLE)].replace("\"", ""); // NON-NLS if (url.isEmpty()) { return null; @@ -801,15 +809,14 @@ final class ExtractEdge extends Extract { * @param programName Name of the module creating the attribute * @return A collection of attributed of a downloaded file */ - private Collection createDownloadAttributes(String path, String url, Long accessTime, String domain, String programName) { + private Collection createDownloadAttributes(String path, Long pathID, String url, Long accessTime, String domain, String programName) { Collection bbattributes = new ArrayList<>(); bbattributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PATH, RecentActivityExtracterModuleFactory.getModuleName(), (path != null) ? path : "")); - long pathID = Util.findID(dataSource, path); - if (pathID != -1) { + if (pathID != null && pathID != -1) { bbattributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PATH_ID, RecentActivityExtracterModuleFactory.getModuleName(), pathID)); @@ -879,7 +886,7 @@ final class ExtractEdge extends Extract { * @return "decoded" string or null if a non-hex value was found */ private String hexToChar(String hexString) { - String[] hexValues = hexString.split(" "); + String[] hexValues = hexString.split(" "); // NON-NLS StringBuilder output = new StringBuilder(); for (String str : hexValues) { @@ -912,7 +919,7 @@ final class ExtractEdge extends Extract { return null; } - String[] tokens = domain.split("\\."); + String[] tokens = domain.split("\\."); // NON-NLS if (tokens.length < 2 || tokens.length > 3) { return domain; // don't know what to do, just send it back as is @@ -1004,7 +1011,7 @@ final class ExtractEdge extends Extract { nameIdx = headers.indexOf(EDGE_HEAD_NAME); idIdx = headers.indexOf(EDGE_HEAD_CONTAINER_ID); } else { - String[] row = line.split(","); + String[] row = line.split(","); // NON-NLS String name = row[nameIdx]; String id = row[idIdx]; From 67ebffac6ca55a2b37deac7897c04f4975a77e16 Mon Sep 17 00:00:00 2001 From: Kelly Kelly Date: Mon, 4 Mar 2019 12:50:01 -0500 Subject: [PATCH 09/15] Removed the plist jar from recentActivity, it should be using the one from corelib now. --- RecentActivity/ivy.xml | 2 +- RecentActivity/nbproject/project.properties | 1 - RecentActivity/nbproject/project.xml | 4 ---- 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/RecentActivity/ivy.xml b/RecentActivity/ivy.xml index ca95f14a98..26621b0320 100644 --- a/RecentActivity/ivy.xml +++ b/RecentActivity/ivy.xml @@ -7,6 +7,6 @@ - + diff --git a/RecentActivity/nbproject/project.properties b/RecentActivity/nbproject/project.properties index 4071f4a54e..9736070e53 100644 --- a/RecentActivity/nbproject/project.properties +++ b/RecentActivity/nbproject/project.properties @@ -1,4 +1,3 @@ -file.reference.dd-plist-1.20.jar=release/modules/ext/dd-plist-1.20.jar javac.source=1.8 javac.compilerargs=-Xlint -Xlint:-serial license.file=../LICENSE-2.0.txt diff --git a/RecentActivity/nbproject/project.xml b/RecentActivity/nbproject/project.xml index 9584170602..87619a8356 100644 --- a/RecentActivity/nbproject/project.xml +++ b/RecentActivity/nbproject/project.xml @@ -74,10 +74,6 @@ - - ext/dd-plist-1.20.jar - release/modules/ext/dd-plist-1.20.jar - From 151e4218dadb37f1a8a4289de341f989ae9f81ff Mon Sep 17 00:00:00 2001 From: Kelly Kelly Date: Mon, 4 Mar 2019 15:57:14 -0500 Subject: [PATCH 10/15] Modify Extract and subclasses to take a progress bar in process function --- .../autopsy/recentactivity/Chrome.java | 26 ++++++++++++++++++- .../DataSourceUsageAnalyzer.java | 7 +++-- .../autopsy/recentactivity/Extract.java | 3 ++- .../autopsy/recentactivity/ExtractEdge.java | 17 +++++++++--- .../autopsy/recentactivity/ExtractIE.java | 20 +++++++++++++- .../autopsy/recentactivity/ExtractOs.java | 7 +++-- .../recentactivity/ExtractRegistry.java | 8 ++++-- .../autopsy/recentactivity/ExtractSafari.java | 15 ++++++++--- .../autopsy/recentactivity/Firefox.java | 25 +++++++++++++++++- .../recentactivity/RAImageIngestModule.java | 2 +- .../recentactivity/RecentDocumentsByLnk.java | 9 ++++++- .../SearchEngineURLQueryAnalyzer.java | 8 ++++-- 12 files changed, 126 insertions(+), 21 deletions(-) diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Chrome.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Chrome.java index 211372d6d0..a30cf2c59a 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Chrome.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Chrome.java @@ -37,11 +37,13 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.FileReader; import java.io.IOException; +import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.autopsy.casemodule.services.FileManager; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.NetworkUtils; +import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProgress; import org.sleuthkit.autopsy.ingest.IngestJobContext; import org.sleuthkit.autopsy.ingest.ModuleDataEvent; import org.sleuthkit.datamodel.AbstractFile; @@ -81,21 +83,43 @@ class Chrome extends Extract { private final Logger logger = Logger.getLogger(this.getClass().getName()); private Content dataSource; private IngestJobContext context; + + @Messages({ + "Progress_Message_Chrome_History=Chrome History", + "Progress_Message_Chrome_Bookmarks=Chrome Bookmarks", + "Progress_Message_Chrome_Cookies=Chrome Cookies", + "Progress_Message_Chrome_Downloads=Chrome Downloads", + "Progress_Message_Chrome_FormHistory=Chrome Form History", + "Progress_Message_Chrome_AutoFill=Chrome Auto Fill", + "Progress_Message_Chrome_Logins=Chrome Logins", + }) Chrome() { moduleName = NbBundle.getMessage(Chrome.class, "Chrome.moduleName"); } @Override - public void process(Content dataSource, IngestJobContext context) { + public void process(Content dataSource, IngestJobContext context, DataSourceIngestModuleProgress progressBar) { this.dataSource = dataSource; this.context = context; dataFound = false; + + progressBar.progress(Bundle.Progress_Message_Chrome_History()); this.getHistory(); + + progressBar.progress(Bundle.Progress_Message_Chrome_Bookmarks()); this.getBookmark(); + + progressBar.progress(Bundle.Progress_Message_Chrome_Cookies()); this.getCookie(); + + progressBar.progress(Bundle.Progress_Message_Chrome_Logins()); this.getLogins(); + + progressBar.progress(Bundle.Progress_Message_Chrome_AutoFill()); this.getAutofill(); + + progressBar.progress(Bundle.Progress_Message_Chrome_Downloads()); this.getDownload(); } diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/DataSourceUsageAnalyzer.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/DataSourceUsageAnalyzer.java index 9c841442a3..8b40633d2b 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/DataSourceUsageAnalyzer.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/DataSourceUsageAnalyzer.java @@ -26,6 +26,7 @@ import org.apache.commons.io.FilenameUtils; import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.casemodule.services.FileManager; import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProgress; import org.sleuthkit.autopsy.ingest.IngestJobContext; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.BlackboardArtifact; @@ -46,12 +47,14 @@ class DataSourceUsageAnalyzer extends Extract { @Messages({ "# {0} - OS name", - "DataSourceUsageAnalyzer.customVolume.label=OS Drive ({0})" + "DataSourceUsageAnalyzer.customVolume.label=OS Drive ({0})", + "Progress_Message_Analyze_Usage=Data Sources Usage Analysis", }) @Override - void process(Content dataSource, IngestJobContext context) { + void process(Content dataSource, IngestJobContext context, DataSourceIngestModuleProgress progressBar) { this.dataSource = dataSource; try { + progressBar.progress(Bundle.Progress_Message_Analyze_Usage()); createDataSourceUsageArtifacts(); } catch (TskCoreException ex) { logger.log(Level.WARNING, "Failed to check if datasource contained a volume with operating system specific files", ex); diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Extract.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Extract.java index b4f04c79cd..3ffbad514b 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Extract.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Extract.java @@ -44,6 +44,7 @@ import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; import org.sleuthkit.autopsy.coreutils.SQLiteDBConnect; import org.sleuthkit.autopsy.datamodel.ContentUtils; +import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProgress; import org.sleuthkit.autopsy.ingest.IngestJobContext; import org.sleuthkit.autopsy.ingest.IngestModule.IngestModuleException; import org.sleuthkit.datamodel.AbstractFile; @@ -85,7 +86,7 @@ abstract class Extract { void configExtractor() throws IngestModuleException { } - abstract void process(Content dataSource, IngestJobContext context); + abstract void process(Content dataSource, IngestJobContext context, DataSourceIngestModuleProgress progressBar); void complete() { } diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractEdge.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractEdge.java index ff793703b1..a446b6450a 100755 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractEdge.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractEdge.java @@ -43,6 +43,7 @@ import org.sleuthkit.autopsy.coreutils.NetworkUtils; import org.sleuthkit.autopsy.coreutils.PlatformUtil; import org.sleuthkit.autopsy.datamodel.ContentUtils; import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProcessTerminator; +import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProgress; import org.sleuthkit.autopsy.ingest.IngestJobContext; import org.sleuthkit.autopsy.ingest.IngestServices; import org.sleuthkit.autopsy.ingest.ModuleDataEvent; @@ -107,6 +108,9 @@ final class ExtractEdge extends Extract { "ExtractEdge_process_errMsg_spartanFail=Failure processing Microsoft Edge spartan.edb file", "ExtractEdge_Module_Name=Microsoft Edge", "ExtractEdge_getHistory_containerFileNotFound=Error while trying to analyze Edge history", + "Progress_Message_Edge_History=Microsoft Edge History", + "Progress_Message_Edge_Bookmarks=Microsoft Edge Bookmarks", + "Progress_Message_Edge_Cookies=Microsoft Edge Cookies", }) /** @@ -122,7 +126,7 @@ final class ExtractEdge extends Extract { } @Override - void process(Content dataSource, IngestJobContext context) { + void process(Content dataSource, IngestJobContext context, DataSourceIngestModuleProgress progressBar) { this.dataSource = dataSource; this.context = context; this.setFoundData(false); @@ -164,12 +168,13 @@ final class ExtractEdge extends Extract { } try { - this.processWebCacheDbFile(esedumper, webCacheFiles); + this.processWebCacheDbFile(esedumper, webCacheFiles, progressBar); } catch (IOException | TskCoreException ex) { this.addErrorMessage(Bundle.ExtractEdge_process_errMsg_webcacheFail()); LOG.log(Level.SEVERE, "Error returned from processWebCacheDbFile", ex); // NON-NLS } + progressBar.progress(Bundle.Progress_Message_Edge_Bookmarks()); try { this.processSpartanDbFile(esedumper, spartanFiles); } catch (IOException | TskCoreException ex) { @@ -187,7 +192,7 @@ final class ExtractEdge extends Extract { * @throws IOException * @throws TskCoreException */ - void processWebCacheDbFile(String eseDumperPath, List webCacheFiles) throws IOException, TskCoreException { + void processWebCacheDbFile(String eseDumperPath, List webCacheFiles, DataSourceIngestModuleProgress progressBar) throws IOException, TskCoreException { for (AbstractFile webCacheFile : webCacheFiles) { @@ -218,13 +223,17 @@ final class ExtractEdge extends Extract { if (context.dataSourceIngestIsCancelled()) { return; } - + + progressBar.progress(Bundle.Progress_Message_Edge_History()); + this.getHistory(webCacheFile, resultsDir); if (context.dataSourceIngestIsCancelled()) { return; } + progressBar.progress(Bundle.Progress_Message_Edge_Cookies()); + this.getCookies(webCacheFile, resultsDir); // if (context.dataSourceIngestIsCancelled()) { diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractIE.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractIE.java index 223ed69799..099579ecae 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractIE.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractIE.java @@ -41,6 +41,7 @@ import java.util.Collection; import java.util.Scanner; import java.util.stream.Collectors; import org.openide.modules.InstalledFileLocator; +import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.autopsy.datamodel.ContentUtils; @@ -53,6 +54,7 @@ import org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE; import org.sleuthkit.datamodel.Content; import org.sleuthkit.autopsy.coreutils.PlatformUtil; import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProcessTerminator; +import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProgress; import org.sleuthkit.autopsy.ingest.IngestJobContext; import org.sleuthkit.datamodel.*; @@ -71,6 +73,16 @@ class ExtractIE extends Extract { private static final SimpleDateFormat dateFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); private Content dataSource; private IngestJobContext context; + + @Messages({ + "Progress_Message_IE_History=IE History", + "Progress_Message_IE_Bookmarks=IE Bookmarks", + "Progress_Message_IE_Cookies=IE Cookies", + "Progress_Message_IE_Downloads=IE Downloads", + "Progress_Message_IE_FormHistory=IE Form History", + "Progress_Message_IE_AutoFill=IE Auto Fill", + "Progress_Message_IE_Logins=IE Logins", + }) ExtractIE() throws NoCurrentCaseException { moduleName = NbBundle.getMessage(ExtractIE.class, "ExtractIE.moduleName.text"); @@ -79,12 +91,18 @@ class ExtractIE extends Extract { } @Override - public void process(Content dataSource, IngestJobContext context) { + public void process(Content dataSource, IngestJobContext context, DataSourceIngestModuleProgress progressBar) { this.dataSource = dataSource; this.context = context; dataFound = false; + + progressBar.progress(Bundle.Progress_Message_IE_Bookmarks()); this.getBookmark(); + + progressBar.progress(Bundle.Progress_Message_IE_Cookies()); this.getCookie(); + + progressBar.progress(Bundle.Progress_Message_IE_History()); this.getHistory(); } diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractOs.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractOs.java index 04976cb95b..9a838d2adb 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractOs.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractOs.java @@ -28,6 +28,7 @@ import org.apache.commons.io.FilenameUtils; import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.casemodule.services.FileManager; import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProgress; import org.sleuthkit.autopsy.ingest.IngestJobContext; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.BlackboardArtifact; @@ -39,7 +40,8 @@ import org.sleuthkit.datamodel.TskCoreException; * Create OS INFO artifacts for the Operating Systems believed to be present on * the data source. */ -@Messages({"ExtractOs.parentModuleName=Recent Activity"}) +@Messages({"ExtractOs.parentModuleName=Recent Activity", + "ExtractOS_progressMessage=Checking for OS"}) class ExtractOs extends Extract { private static final Logger logger = Logger.getLogger(ExtractOs.class.getName()); @@ -64,9 +66,10 @@ class ExtractOs extends Extract { private Content dataSource; @Override - void process(Content dataSource, IngestJobContext context) { + void process(Content dataSource, IngestJobContext context, DataSourceIngestModuleProgress progressBar) { this.dataSource = dataSource; try { + progressBar.progress(Bundle.ExtractOS_progressMessage()); for (OS_TYPE value : OS_TYPE.values()) { checkForOSFiles(value); } diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java index 68885bcc52..77338655a5 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java @@ -52,6 +52,7 @@ import org.xml.sax.SAXException; import java.nio.file.Path; import static java.util.TimeZone.getTimeZone; import org.openide.util.Lookup; +import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProgress; import org.sleuthkit.autopsy.ingest.IngestModule.IngestModuleException; import org.sleuthkit.autopsy.ingest.IngestServices; import org.sleuthkit.autopsy.ingest.ModuleDataEvent; @@ -66,7 +67,8 @@ import org.sleuthkit.datamodel.ReadContentInputStream.ReadContentInputStreamExce */ @NbBundle.Messages({ "RegRipperNotFound=Autopsy RegRipper executable not found.", - "RegRipperFullNotFound=Full version RegRipper executable not found." + "RegRipperFullNotFound=Full version RegRipper executable not found.", + "Progress_Message_Analyze_Registry=Analyzing Registry Files" }) class ExtractRegistry extends Extract { @@ -969,9 +971,11 @@ class ExtractRegistry extends Extract { } @Override - public void process(Content dataSource, IngestJobContext context) { + public void process(Content dataSource, IngestJobContext context, DataSourceIngestModuleProgress progressBar) { this.dataSource = dataSource; this.context = context; + + progressBar.progress(Bundle.Progress_Message_Analyze_Registry()); analyzeRegistryFiles(); } diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractSafari.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractSafari.java index afb52654b2..3f98bcdf91 100755 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractSafari.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractSafari.java @@ -41,6 +41,7 @@ import org.sleuthkit.autopsy.casemodule.services.FileManager; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.NetworkUtils; import org.sleuthkit.autopsy.datamodel.ContentUtils; +import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProgress; import org.sleuthkit.autopsy.ingest.IngestJobContext; import org.sleuthkit.autopsy.ingest.IngestServices; import org.sleuthkit.autopsy.ingest.ModuleDataEvent; @@ -89,6 +90,10 @@ final class ExtractSafari extends Extract { "ExtractSafari_Error_Getting_History=An error occurred while processing Safari history files.", "ExtractSafari_Error_Parsing_Bookmark=An error occured while processing Safari Bookmark files", "ExtractSafari_Error_Parsing_Cookies=An error occured while processing Safari Cookies files", + "Progress_Message_Safari_History=Safari History", + "Progress_Message_Safari_Bookmarks=Safari Bookmarks", + "Progress_Message_Safari_Cookies=Safari Cookies", + "Progress_Message_Safari_Downloads=Safari Downloads", }) /** @@ -105,9 +110,10 @@ final class ExtractSafari extends Extract { } @Override - void process(Content dataSource, IngestJobContext context) { + void process(Content dataSource, IngestJobContext context, DataSourceIngestModuleProgress progressBar) { setFoundData(false); - + + progressBar.progress(Bundle.Progress_Message_Safari_Cookies()); try { processHistoryDB(dataSource, context); @@ -116,6 +122,7 @@ final class ExtractSafari extends Extract { LOG.log(Level.SEVERE, "Exception thrown while processing history file: {0}", ex); //NON-NLS } + progressBar.progress(Bundle.Progress_Message_Safari_Bookmarks()); try { processBookmarkPList(dataSource, context); } catch (IOException | TskCoreException | SAXException | PropertyListFormatException | ParseException | ParserConfigurationException ex) { @@ -123,13 +130,15 @@ final class ExtractSafari extends Extract { LOG.log(Level.SEVERE, "Exception thrown while parsing Safari Bookmarks file: {0}", ex); //NON-NLS } - try { + progressBar.progress(Bundle.Progress_Message_Safari_Downloads()); + try { processDownloadsPList(dataSource, context); } catch (IOException | TskCoreException | SAXException | PropertyListFormatException | ParseException | ParserConfigurationException ex) { this.addErrorMessage(Bundle.ExtractSafari_Error_Parsing_Bookmark()); LOG.log(Level.SEVERE, "Exception thrown while parsing Safari Download.plist file: {0}", ex); //NON-NLS } + progressBar.progress(Bundle.Progress_Message_Safari_Cookies()); try { processBinaryCookieFile(dataSource, context); } catch (IOException | TskCoreException ex) { diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Firefox.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Firefox.java index 043e0bf0ff..129d86d10d 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Firefox.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Firefox.java @@ -44,12 +44,14 @@ import java.util.Set; import java.util.logging.Level; import org.openide.util.NbBundle; +import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.autopsy.casemodule.services.FileManager; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.NetworkUtils; import org.sleuthkit.autopsy.datamodel.ContentUtils; +import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProgress; import org.sleuthkit.autopsy.ingest.IngestJobContext; import org.sleuthkit.autopsy.ingest.IngestServices; import org.sleuthkit.autopsy.ingest.ModuleDataEvent; @@ -63,6 +65,15 @@ import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.ReadContentInputStream.ReadContentInputStreamException; import org.sleuthkit.datamodel.TskCoreException; +@Messages({ + "Progress_Message_Firefox_History=Firefox History", + "Progress_Message_Firefox_Bookmarks=Firefox Bookmarks", + "Progress_Message_Firefox_Cookies=Firefox Cookies", + "Progress_Message_Firefox_Downloads=Firefox Downloads", + "Progress_Message_Firefox_FormHistory=Firefox Form History", + "Progress_Message_Firefox_AutoFill=Firefox Auto Fill" +}) + /** * Firefox recent activity extraction */ @@ -95,15 +106,27 @@ class Firefox extends Extract { } @Override - public void process(Content dataSource, IngestJobContext context) { + public void process(Content dataSource, IngestJobContext context, DataSourceIngestModuleProgress progressBar) { this.dataSource = dataSource; this.context = context; dataFound = false; + + progressBar.progress(Bundle.Progress_Message_Firefox_History()); this.getHistory(); + + progressBar.progress(Bundle.Progress_Message_Firefox_Bookmarks()); this.getBookmark(); + + progressBar.progress(Bundle.Progress_Message_Firefox_Downloads()); this.getDownload(); + + progressBar.progress(Bundle.Progress_Message_Firefox_Cookies()); this.getCookie(); + + progressBar.progress(Bundle.Progress_Message_Firefox_FormHistory()); this.getFormsHistory(); + + progressBar.progress(Bundle.Progress_Message_Firefox_AutoFill()); this.getAutofillProfiles(); } diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RAImageIngestModule.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RAImageIngestModule.java index 434cf968de..18757cc997 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RAImageIngestModule.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RAImageIngestModule.java @@ -119,7 +119,7 @@ public final class RAImageIngestModule implements DataSourceIngestModule { progressBar.progress(extracter.getName(), i); try { - extracter.process(dataSource, context); + extracter.process(dataSource, context, progressBar); } catch (Exception ex) { logger.log(Level.SEVERE, "Exception occurred in " + extracter.getName(), ex); //NON-NLS subCompleted.append(NbBundle.getMessage(this.getClass(), "RAImageIngestModule.process.errModFailed", diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RecentDocumentsByLnk.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RecentDocumentsByLnk.java index bb200551fb..160f88f9b7 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RecentDocumentsByLnk.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RecentDocumentsByLnk.java @@ -29,6 +29,7 @@ import java.util.logging.Level; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.coreutils.Logger; import java.util.Collection; +import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.coreutils.JLNK; import org.sleuthkit.autopsy.coreutils.JLnkParser; import org.sleuthkit.autopsy.coreutils.JLnkParserException; @@ -53,6 +54,10 @@ class RecentDocumentsByLnk extends Extract { private IngestServices services = IngestServices.getInstance(); private Content dataSource; private IngestJobContext context; + + @Messages({ + "Progress_Message_Extract_Resent_Docs=Recent Documents", + }) /** * Find the documents that Windows stores about recent documents and make @@ -125,10 +130,12 @@ class RecentDocumentsByLnk extends Extract { } @Override - public void process(Content dataSource, IngestJobContext context) { + public void process(Content dataSource, IngestJobContext context, DataSourceIngestModuleProgress progressBar) { this.dataSource = dataSource; this.context = context; dataFound = false; + + progressBar.progress(Bundle.Progress_Message_Extract_Resent_Docs()); this.getRecentDocuments(); } } diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java index 4c8999ec47..5aff1a4a0b 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java @@ -33,6 +33,7 @@ import org.openide.util.NbBundle; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.PlatformUtil; import org.sleuthkit.autopsy.coreutils.XMLUtil; +import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProgress; import org.sleuthkit.autopsy.ingest.IngestJobContext; import org.sleuthkit.autopsy.ingest.IngestModule.IngestModuleException; import org.sleuthkit.autopsy.ingest.IngestServices; @@ -62,7 +63,8 @@ import org.xml.sax.SAXException; "cannotBuildXmlParser=Unable to build XML parser: ", "cannotLoadSEUQA=Unable to load Search Engine URL Query Analyzer settings file, SEUQAMappings.xml: ", "cannotParseXml=Unable to parse XML file: ", - "# {0} - file name", "SearchEngineURLQueryAnalyzer.init.exception.msg=Unable to find {0}." + "# {0} - file name", "SearchEngineURLQueryAnalyzer.init.exception.msg=Unable to find {0}.", + "Progress_Message_Find_Search_Query=Find Search Queries" }) class SearchEngineURLQueryAnalyzer extends Extract { @@ -396,9 +398,11 @@ class SearchEngineURLQueryAnalyzer extends Extract { } @Override - public void process(Content dataSource, IngestJobContext context) { + public void process(Content dataSource, IngestJobContext context, DataSourceIngestModuleProgress progressBar) { this.dataSource = dataSource; this.context = context; + + progressBar.progress(Bundle.Progress_Message_Find_Search_Query()); this.findSearchQueries(); logger.log(Level.INFO, "Search Engine stats: \n{0}", getTotals()); //NON-NLS } From a82bcc603e8b052b972ac345edd67119a502f0b1 Mon Sep 17 00:00:00 2001 From: Kelly Kelly Date: Tue, 5 Mar 2019 10:33:44 -0500 Subject: [PATCH 11/15] Modified code based on review comments --- .../autopsy/recentactivity/ExtractEdge.java | 31 ++++++------------- 1 file changed, 9 insertions(+), 22 deletions(-) diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractEdge.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractEdge.java index 9ea55287ff..76adb9a11a 100755 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractEdge.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractEdge.java @@ -38,6 +38,7 @@ import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.autopsy.coreutils.ExecUtil; +import org.sleuthkit.autopsy.coreutils.FileUtil; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.NetworkUtils; import org.sleuthkit.autopsy.coreutils.PlatformUtil; @@ -181,8 +182,8 @@ final class ExtractEdge extends Extract { } /** - * Dump the tables from WebCacheV01.dat and look for the data contained with - * in those files including downloads, cookies and history. + * Process WebCacheV01.dat ese database file creating artifacts for cookies, + * and history contained within. * * @param eseDumperPath Path to ESEDatabaseView.exe * @param webCacheFiles List of case WebCacheV01.dat files @@ -231,21 +232,14 @@ final class ExtractEdge extends Extract { } finally { tempWebCacheFile.delete(); - - // Emppty the result dir - File[] resultFiles = resultsDir.listFiles(); - for (File file : resultFiles) { - file.delete(); - } - - resultsDir.delete(); + FileUtil.deleteFileDir(resultsDir); } } } /** - * Creates a temp version of the database and runs the ESEDatabaseView tool - * to dump each of the database tables into a temporary folder. + * Process spartan.edb ese database file creating artifacts for the bookmarks + * contained within. * * @param eseDumperPath Path to ESEDatabaseViewer * @param spartanFiles List of the case spartan.edb files @@ -286,14 +280,7 @@ final class ExtractEdge extends Extract { } finally { tempSpartanFile.delete(); - - // Empty the result dir - File[] resultFiles = resultsDir.listFiles(); - for (File file : resultFiles) { - file.delete(); - } - - resultsDir.delete(); + FileUtil.deleteFileDir(resultsDir); } } } @@ -536,7 +523,7 @@ final class ExtractEdge extends Extract { private List fetchWebCacheDBFiles() throws TskCoreException { org.sleuthkit.autopsy.casemodule.services.FileManager fileManager = currentCase.getServices().getFileManager(); - return fileManager.findFiles(dataSource, EDGE_WEBCACHE_NAME, EDGE_WEBCACHE_FOLDER_NAME); //NON-NLS + return fileManager.findFiles(dataSource, EDGE_WEBCACHE_NAME, EDGE_WEBCACHE_FOLDER_NAME); } /** @@ -548,7 +535,7 @@ final class ExtractEdge extends Extract { private List fetchSpartanDBFiles() throws TskCoreException { org.sleuthkit.autopsy.casemodule.services.FileManager fileManager = currentCase.getServices().getFileManager(); - return fileManager.findFiles(dataSource, EDGE_SPARTAN_NAME, EDGE_SPARTAN_FOLDER_NAME); //NON-NLS + return fileManager.findFiles(dataSource, EDGE_SPARTAN_NAME, EDGE_SPARTAN_FOLDER_NAME); } /** From 6b6f38d5502d8932e0d17d7147c208b2bcdd2c2b Mon Sep 17 00:00:00 2001 From: rcordovano Date: Tue, 5 Mar 2019 14:51:02 -0500 Subject: [PATCH 12/15] Fix cause of compile warning for DataContentViewerOtherCases.java --- .../contentviewer/DataContentViewerOtherCases.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/contentviewer/DataContentViewerOtherCases.java b/Core/src/org/sleuthkit/autopsy/centralrepository/contentviewer/DataContentViewerOtherCases.java index 4762da103a..308e14be63 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/contentviewer/DataContentViewerOtherCases.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/contentviewer/DataContentViewerOtherCases.java @@ -380,7 +380,7 @@ public class DataContentViewerOtherCases extends JPanel implements DataContentVi * a case to be unique. We should improve this in the future. */ Set cases = new HashSet<>(); - Map devices = new HashMap(); + Map devices = new HashMap<>(); for (int i=0; i < model.getRowCount(); i++) { String caseName = (String) model.getValueAt(i, caseColumnIndex); From 2b739fe3135cc64733cf842d36815894dde75d31 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Tue, 5 Mar 2019 15:14:26 -0500 Subject: [PATCH 13/15] Bundle.properties-MERGE file fixes --- .../sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED | 2 +- .../sleuthkit/autopsy/recentactivity/Bundle.properties-MERGED | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED index f4febc1d7c..61c0d2d2c7 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED @@ -34,7 +34,7 @@ KeywordSearchIngestModule.startupMessage.failedToGetIndexSchema=Failed to get sc KeywordSearchResultFactory.createNodeForKey.noResultsFound.text=No results found. KeywordSearchResultFactory.query.exception.msg=Could not perform the query OpenIDE-Module-Display-Category=Ingest Module -OpenIDE-Module-Long-Description=Keyword Search ingest module.\n\n\The module indexes files found in the disk image at ingest time.\n\It then periodically runs the search on the indexed files using one or more keyword lists (containing pure words and/or regular expressions) and posts results.\n\n\The module also contains additional tools integrated in the main GUI, such as keyword list configuration, keyword seach bar in the top-right corner, extracted text viewer and search results viewer showing highlighted keywords found. +OpenIDE-Module-Long-Description=Keyword Search ingest module.\n\nThe module indexes files found in the disk image at ingest time.\nIt then periodically runs the search on the indexed files using one or more keyword lists (containing pure words and/or regular expressions) and posts results.\n\nThe module also contains additional tools integrated in the main GUI, such as keyword list configuration, keyword seach bar in the top-right corner, extracted text viewer and search results viewer showing highlighted keywords found. OpenIDE-Module-Name=KeywordSearch OptionsCategory_Name_KeywordSearchOptions=Keyword Search OptionsCategory_Keywords_KeywordSearchOptions=Keyword Search diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Bundle.properties-MERGED b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Bundle.properties-MERGED index 58ea431f13..5b275de006 100755 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Bundle.properties-MERGED +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Bundle.properties-MERGED @@ -48,7 +48,7 @@ ExtractSafari_Error_Parsing_Bookmark=An error occured while processing Safari Bo ExtractSafari_Error_Parsing_Cookies=An error occured while processing Safari Cookies files ExtractSafari_Module_Name=Safari OpenIDE-Module-Display-Category=Ingest Module -OpenIDE-Module-Long-Description=Recent Activity ingest module.\n\n\The module extracts useful information about the recent user activity on the disk image being ingested, such as:\n\n- Recently open documents,\n- Web acitivity (sites visited, stored cookies, bookmarked sites, search engine queries, file downloads),\n- Recently attached devices,\n- Installed programs.\n\n\The module currently supports Windows only disk images.\n\The plugin is also fully functional when deployed on Windows version of Autopsy. +OpenIDE-Module-Long-Description=Recent Activity ingest module.\n\nThe module extracts useful information about the recent user activity on the disk image being ingested, such as:\n\n- Recently open documents,\n- Web acitivity (sites visited, stored cookies, bookmarked sites, search engine queries, file downloads),\n- Recently attached devices,\n- Installed programs.\n\nThe module currently supports Windows only disk images.\nThe plugin is also fully functional when deployed on Windows version of Autopsy. OpenIDE-Module-Name=RecentActivity OpenIDE-Module-Short-Description=Recent Activity finder ingest module Chrome.moduleName=Chrome @@ -146,7 +146,7 @@ SearchEngineURLQueryAnalyzer.init.exception.msg=Unable to find {0}. SearchEngineURLQueryAnalyzer.moduleName.text=Search Engine SearchEngineURLQueryAnalyzer.engineName.none=NONE SearchEngineURLQueryAnalyzer.domainSubStr.none=NONE -SearchEngineURLQueryAnalyzer.toString=Name: {0}\nDomain Substring: {1}\n\count: {2}\nSplit Tokens: \n{3} +SearchEngineURLQueryAnalyzer.toString=Name: {0}\nDomain Substring: {1}\ncount: {2}\nSplit Tokens: \n{3} SearchEngineURLQueryAnalyzer.parentModuleName.noSpace=RecentActivity SearchEngineURLQueryAnalyzer.parentModuleName=Recent Activity UsbDeviceIdMapper.parseAndLookup.text=Product: {0} From 706005772fe99075d1c32ab0e707798649d1bbea Mon Sep 17 00:00:00 2001 From: Kelly Kelly Date: Tue, 5 Mar 2019 16:12:58 -0500 Subject: [PATCH 14/15] removed commented out line from ivy.xml --- .../sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED | 2 +- RecentActivity/ivy.xml | 3 --- .../sleuthkit/autopsy/recentactivity/Bundle.properties-MERGED | 4 ++-- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED index 61c0d2d2c7..f4febc1d7c 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED @@ -34,7 +34,7 @@ KeywordSearchIngestModule.startupMessage.failedToGetIndexSchema=Failed to get sc KeywordSearchResultFactory.createNodeForKey.noResultsFound.text=No results found. KeywordSearchResultFactory.query.exception.msg=Could not perform the query OpenIDE-Module-Display-Category=Ingest Module -OpenIDE-Module-Long-Description=Keyword Search ingest module.\n\nThe module indexes files found in the disk image at ingest time.\nIt then periodically runs the search on the indexed files using one or more keyword lists (containing pure words and/or regular expressions) and posts results.\n\nThe module also contains additional tools integrated in the main GUI, such as keyword list configuration, keyword seach bar in the top-right corner, extracted text viewer and search results viewer showing highlighted keywords found. +OpenIDE-Module-Long-Description=Keyword Search ingest module.\n\n\The module indexes files found in the disk image at ingest time.\n\It then periodically runs the search on the indexed files using one or more keyword lists (containing pure words and/or regular expressions) and posts results.\n\n\The module also contains additional tools integrated in the main GUI, such as keyword list configuration, keyword seach bar in the top-right corner, extracted text viewer and search results viewer showing highlighted keywords found. OpenIDE-Module-Name=KeywordSearch OptionsCategory_Name_KeywordSearchOptions=Keyword Search OptionsCategory_Keywords_KeywordSearchOptions=Keyword Search diff --git a/RecentActivity/ivy.xml b/RecentActivity/ivy.xml index 26621b0320..290c8371ea 100644 --- a/RecentActivity/ivy.xml +++ b/RecentActivity/ivy.xml @@ -6,7 +6,4 @@ - - - diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Bundle.properties-MERGED b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Bundle.properties-MERGED index 5b275de006..58ea431f13 100755 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Bundle.properties-MERGED +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Bundle.properties-MERGED @@ -48,7 +48,7 @@ ExtractSafari_Error_Parsing_Bookmark=An error occured while processing Safari Bo ExtractSafari_Error_Parsing_Cookies=An error occured while processing Safari Cookies files ExtractSafari_Module_Name=Safari OpenIDE-Module-Display-Category=Ingest Module -OpenIDE-Module-Long-Description=Recent Activity ingest module.\n\nThe module extracts useful information about the recent user activity on the disk image being ingested, such as:\n\n- Recently open documents,\n- Web acitivity (sites visited, stored cookies, bookmarked sites, search engine queries, file downloads),\n- Recently attached devices,\n- Installed programs.\n\nThe module currently supports Windows only disk images.\nThe plugin is also fully functional when deployed on Windows version of Autopsy. +OpenIDE-Module-Long-Description=Recent Activity ingest module.\n\n\The module extracts useful information about the recent user activity on the disk image being ingested, such as:\n\n- Recently open documents,\n- Web acitivity (sites visited, stored cookies, bookmarked sites, search engine queries, file downloads),\n- Recently attached devices,\n- Installed programs.\n\n\The module currently supports Windows only disk images.\n\The plugin is also fully functional when deployed on Windows version of Autopsy. OpenIDE-Module-Name=RecentActivity OpenIDE-Module-Short-Description=Recent Activity finder ingest module Chrome.moduleName=Chrome @@ -146,7 +146,7 @@ SearchEngineURLQueryAnalyzer.init.exception.msg=Unable to find {0}. SearchEngineURLQueryAnalyzer.moduleName.text=Search Engine SearchEngineURLQueryAnalyzer.engineName.none=NONE SearchEngineURLQueryAnalyzer.domainSubStr.none=NONE -SearchEngineURLQueryAnalyzer.toString=Name: {0}\nDomain Substring: {1}\ncount: {2}\nSplit Tokens: \n{3} +SearchEngineURLQueryAnalyzer.toString=Name: {0}\nDomain Substring: {1}\n\count: {2}\nSplit Tokens: \n{3} SearchEngineURLQueryAnalyzer.parentModuleName.noSpace=RecentActivity SearchEngineURLQueryAnalyzer.parentModuleName=Recent Activity UsbDeviceIdMapper.parseAndLookup.text=Product: {0} From ac754393e23c7e49c7dd6a1e5c8c7e26d25f1724 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Tue, 5 Mar 2019 16:33:46 -0500 Subject: [PATCH 15/15] Fix MERGED bundle for KWS, update MERGED bundle for RA --- .../autopsy/keywordsearch/Bundle.properties | 2 +- .../keywordsearch/Bundle.properties-MERGED | 2 +- .../recentactivity/Bundle.properties-MERGED | 32 +++++++++++++++++++ 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties index 9d82d32924..67e6496584 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties @@ -1,5 +1,5 @@ OpenIDE-Module-Display-Category=Ingest Module -OpenIDE-Module-Long-Description=Keyword Search ingest module.\n\n\The module indexes files found in the disk image at ingest time.\n\It then periodically runs the search on the indexed files using one or more keyword lists (containing pure words and/or regular expressions) and posts results.\n\n\The module also contains additional tools integrated in the main GUI, such as keyword list configuration, keyword seach bar in the top-right corner, extracted text viewer and search results viewer showing highlighted keywords found. +OpenIDE-Module-Long-Description=Keyword Search ingest module.\n\nThe module indexes files found in the disk image at ingest time.\nIt then periodically runs the search on the indexed files using one or more keyword lists (containing pure words and/or regular expressions) and posts results.\n\n\The module also contains additional tools integrated in the main GUI, such as keyword list configuration, keyword search bar in the top-right corner, extracted text viewer and search results viewer showing highlighted keywords found. OpenIDE-Module-Name=KeywordSearch OptionsCategory_Name_KeywordSearchOptions=Keyword Search OptionsCategory_Keywords_KeywordSearchOptions=Keyword Search diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED index f4febc1d7c..034e4355d3 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED @@ -34,7 +34,7 @@ KeywordSearchIngestModule.startupMessage.failedToGetIndexSchema=Failed to get sc KeywordSearchResultFactory.createNodeForKey.noResultsFound.text=No results found. KeywordSearchResultFactory.query.exception.msg=Could not perform the query OpenIDE-Module-Display-Category=Ingest Module -OpenIDE-Module-Long-Description=Keyword Search ingest module.\n\n\The module indexes files found in the disk image at ingest time.\n\It then periodically runs the search on the indexed files using one or more keyword lists (containing pure words and/or regular expressions) and posts results.\n\n\The module also contains additional tools integrated in the main GUI, such as keyword list configuration, keyword seach bar in the top-right corner, extracted text viewer and search results viewer showing highlighted keywords found. +OpenIDE-Module-Long-Description=Keyword Search ingest module.\n\nThe module indexes files found in the disk image at ingest time.\nIt then periodically runs the search on the indexed files using one or more keyword lists (containing pure words and/or regular expressions) and posts results.\n\n\The module also contains additional tools integrated in the main GUI, such as keyword list configuration, keyword search bar in the top-right corner, extracted text viewer and search results viewer showing highlighted keywords found. OpenIDE-Module-Name=KeywordSearch OptionsCategory_Name_KeywordSearchOptions=Keyword Search OptionsCategory_Keywords_KeywordSearchOptions=Keyword Search diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Bundle.properties-MERGED b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Bundle.properties-MERGED index 58ea431f13..937b659039 100755 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Bundle.properties-MERGED +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Bundle.properties-MERGED @@ -43,6 +43,7 @@ ExtractOs.unitedLinuxVolume.label=OS Drive (Linux United Linux) ExtractOs.windowsVolume.label=OS Drive (Windows) ExtractOs.yellowDogLinuxOs.label=Linux (Yellow Dog) ExtractOs.yellowDogLinuxVolume.label=OS Drive (Linux Yellow Dog) +ExtractOS_progressMessage=Checking for OS ExtractSafari_Error_Getting_History=An error occurred while processing Safari history files. ExtractSafari_Error_Parsing_Bookmark=An error occured while processing Safari Bookmark files ExtractSafari_Error_Parsing_Cookies=An error occured while processing Safari Cookies files @@ -119,6 +120,37 @@ Firefox.getDlPre24.errMsg.errParsingArtifacts={0}: Error parsing {1} Firefox web Firefox.getDlV24.errMsg.errFetchFiles=Error fetching 'downloads' files for Firefox. Firefox.getDlV24.errMsg.errAnalyzeFile={0}: Error while trying to analyze file:{1} Firefox.getDlV24.errMsg.errParsingArtifacts={0}: Error parsing {1} Firefox web download artifacts. +Progress_Message_Analyze_Registry=Analyzing Registry Files +Progress_Message_Analyze_Usage=Data Sources Usage Analysis +Progress_Message_Chrome_AutoFill=Chrome Auto Fill +Progress_Message_Chrome_Bookmarks=Chrome Bookmarks +Progress_Message_Chrome_Cookies=Chrome Cookies +Progress_Message_Chrome_Downloads=Chrome Downloads +Progress_Message_Chrome_FormHistory=Chrome Form History +Progress_Message_Chrome_History=Chrome History +Progress_Message_Chrome_Logins=Chrome Logins +Progress_Message_Edge_Bookmarks=Microsoft Edge Bookmarks +Progress_Message_Edge_Cookies=Microsoft Edge Cookies +Progress_Message_Edge_History=Microsoft Edge History +Progress_Message_Extract_Resent_Docs=Recent Documents +Progress_Message_Find_Search_Query=Find Search Queries +Progress_Message_Firefox_AutoFill=Firefox Auto Fill +Progress_Message_Firefox_Bookmarks=Firefox Bookmarks +Progress_Message_Firefox_Cookies=Firefox Cookies +Progress_Message_Firefox_Downloads=Firefox Downloads +Progress_Message_Firefox_FormHistory=Firefox Form History +Progress_Message_Firefox_History=Firefox History +Progress_Message_IE_AutoFill=IE Auto Fill +Progress_Message_IE_Bookmarks=IE Bookmarks +Progress_Message_IE_Cookies=IE Cookies +Progress_Message_IE_Downloads=IE Downloads +Progress_Message_IE_FormHistory=IE Form History +Progress_Message_IE_History=IE History +Progress_Message_IE_Logins=IE Logins +Progress_Message_Safari_Bookmarks=Safari Bookmarks +Progress_Message_Safari_Cookies=Safari Cookies +Progress_Message_Safari_Downloads=Safari Downloads +Progress_Message_Safari_History=Safari History RAImageIngestModule.process.started=Started {0} RAImageIngestModule.process.errModFailed={0} failed - see log for details
RAImageIngestModule.process.errModErrs={0} had errors -- see log