Merge branch '4388-implement-sqlite-streaming-design' into 4389-sqlite-stream-impl

Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReader.java

@@ -16,7 +16,6 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
 package org.sleuthkit.autopsy.coreutils;
 
 import java.io.File;
@@ -42,13 +41,35 @@ import org.sleuthkit.datamodel.SleuthkitCase;
 import org.sleuthkit.datamodel.TskCoreException;
 
 /**
+ * Reads row by row through SQLite tables and performs user-defined actions on the row values.
+ * Table values are processed by data type. Users configure these actions for certain data types 
+ * in the Builder. Example usage:
  * 
- * @author dsmyda
+ *  SQLiteTableReader reader = new SQLiteTableReader.Builder(file)
+ *                   .onInteger((i) -> {
+ *                          System.out.println(i);
+ *                   }).build(); 
+ *  reader.read(tableName);
+ * 
+ *  or
+ * 
+ *  SQLiteTableReader reader = new SQLiteTableReader.Builder(file)
+ *                   .onInteger(new Consumer<Integer>() {
+ *                      @Override
+ *                      public void accept(Integer i) {
+ *                          System.out.println(i);
+ *                      }
+ *                   }).build();
+ *  reader.reader(tableName);
+ * 
+ * Invocation of read(String tableName) causes that table name to be processed row by row.
+ * When an Integer is encountered, its value will be passed to the Consumer that 
+ * was defined above.
  */
 public class SQLiteTableReader implements AutoCloseable {
 
     /**
-     * 
+     * Builder patten for configuring SQLiteTableReader instances.
      */
     public static class Builder {
 
@@ -63,7 +84,7 @@ public class SQLiteTableReader implements AutoCloseable {
         private Consumer<Object> forAllAction;
 
         /**
-         * Creates a SQLiteTableReaderBuilder for this abstract file.
+         * Creates a Builder for this abstract file.
          *
          * @param file
          */
@@ -72,12 +93,12 @@ public class SQLiteTableReader implements AutoCloseable {
         }
 
         /**
-         * Specify a function to handle MetaData parsing. The MetaData object
-         * will be parsed before any contents are read from the table.
+         * Specify a function to do on column names. Column names will be read
+         * from left to right.
          *
-         * @param action
+         * @param action Consumer of column name strings
          *
-         * @return
+         * @return Builder reference
          */
         public Builder onColumnNames(Consumer<String> action) {
             this.onColumnNameAction = action;
@@ -85,12 +106,12 @@ public class SQLiteTableReader implements AutoCloseable {
         }
 
         /**
-         * Specify a function to do on receiving a database entry that is type
-         * String.
+         * Specify a function to do when encountering a database value that is
+         * of java type String.
          *
-         * @param action
+         * @param action Consumer of strings
          *
-         * @return
+         * @return Builder reference
          */
         public Builder onString(Consumer<String> action) {
             this.onStringAction = action;
@@ -98,12 +119,12 @@ public class SQLiteTableReader implements AutoCloseable {
         }
 
         /**
-         * Specify a function to do on receiving a database entry that is type
-         * Integer.
+         * Specify a function to do when encountering a database value that is
+         * of java type Integer.
          *
-         * @param action
+         * @param action Consumer of integer
          *
-         * @return
+         * @return Builder reference
          */
         public Builder onInteger(Consumer<Integer> action) {
             this.onIntegerAction = action;
@@ -111,12 +132,12 @@ public class SQLiteTableReader implements AutoCloseable {
         }
 
         /**
-         * Specify a function to do on receiving a database entry that is type
-         * Real.
+         * Specify a function to do when encountering a database value that is
+         * of java type Double.
          *
-         * @param action
+         * @param action Consumer of doubles
          *
-         * @return
+         * @return Builder reference
          */
         public Builder onFloat(Consumer<Double> action) {
             this.onFloatAction = action;
@@ -124,9 +145,12 @@ public class SQLiteTableReader implements AutoCloseable {
         }
 
         /**
+         * Specify a function to do when encountering a database value that is
+         * of java type Long.
          *
-         * @param action
-         * @return 
+         * @param action Consumer of longs
+         *
+         * @return Builder reference
          */
         public Builder onLong(Consumer<Long> action) {
             this.onLongAction = action;
@@ -134,9 +158,12 @@ public class SQLiteTableReader implements AutoCloseable {
         }
 
         /**
+         * Specify a function to do when encountering a database value that is
+         * of java type byte[] aka blob.
          *
-         * @param action
-         * @return 
+         * @param action Consumer of blobs
+         *
+         * @return Builder reference
          */
         public Builder onBlob(Consumer<byte[]> action) {
             this.onBlobAction = action;
@@ -144,11 +171,13 @@ public class SQLiteTableReader implements AutoCloseable {
         }
 
         /**
-         * Specify a function to do for any database entry, regardless of type.
+         * Specify a function to do when encountering any database value,
+         * regardless of type. This function only captures database values, not
+         * column names.
          *
-         * @param action
+         * @param action Consumer of objects
          *
-         * @return
+         * @return Builder reference
          */
         public Builder forAll(Consumer<Object> action) {
             this.forAllAction = action;
@@ -156,10 +185,10 @@ public class SQLiteTableReader implements AutoCloseable {
         }
 
         /**
-         * Pass all params to the SQLTableStream so that it can iterate through
-         * the table
+         * Creates a SQLiteTableReader instance given this Builder
+         * configuration.
          *
-         * @return
+         * @return SQLiteTableReader instance
          */
         public SQLiteTableReader build() {
             return new SQLiteTableReader(this);
@@ -182,20 +211,15 @@ public class SQLiteTableReader implements AutoCloseable {
 
     //Iteration state variables
     private Integer currRowColumnIndex;
-    private boolean unfinishedRowState;
     private Integer columnNameIndex;
     private Integer currentColumnCount;
     private ResultSetMetaData currentMetadata;
 
-    private boolean isFinished;
-    private boolean hasOpened;
+    private boolean liveResultSet;
     private String prevTableName;
 
-    private final BooleanSupplier alwaysFalseCondition = () -> {return false;};
-
     /**
-     * Initialize a new table stream given the parameters passed in from the
-     * StreamBuilder above.
+     * Assigns references to each action based on the Builder configuration.
      */
     private SQLiteTableReader(Builder builder) {
 
@@ -210,65 +234,79 @@ public class SQLiteTableReader implements AutoCloseable {
         this.file = builder.file;
     }
 
+    /**
+     * Ensures the action is null safe. If action is left null, then during
+     * iteration null checks would be necessary. To mitigate against that, no-op
+     * lambdas are substituted for null values.
+     *
+     * @param <T>    Generic type of consumer
+     * @param action Consumer for generic type, supplied by Builder.
+     *
+     * @return If action is null, then a no-op lambda, if not then the action
+     *         itself.
+     */
     private <T> Consumer<T> nonNullValue(Consumer<T> action) {
-        if (Objects.nonNull(action)) {
-            return action;
-        }
-
-        //No-op lambda, keep from NPE or having to check during iteration 
-        //if action == null.
-        return (NO_OP) -> {};
+        return (Objects.nonNull(action)) ? action : NO_OP -> {
+        };
     }
 
     /**
-     * Get table names from database
+     * Fetches all table names from the database.
      *
-     * @return
-     * @throws org.sleuthkit.autopsy.coreutils.SQLiteTableReaderException
+     * @return List of all table names found while querying the sqlite_master
+     *         table
      *
+     * @throws SQLiteTableReaderException
      */
     public List<String> getTableNames() throws SQLiteTableReaderException {
         ensureOpen();
-        
-        List<String> tableNames = new ArrayList<>();
-        
         try (ResultSet tableNameResult = conn.createStatement()
                 .executeQuery("SELECT name FROM sqlite_master "
-                + " WHERE type= 'table' ")) {
+                        + " WHERE type= 'table' ")) {
+            List<String> tableNames = new ArrayList<>();
             while (tableNameResult.next()) {
                 tableNames.add(tableNameResult.getString("name")); //NON-NLS
             }
+            return tableNames;
         } catch (SQLException ex) {
             throw new SQLiteTableReaderException(ex);
         }
-
-        return tableNames;
     }
 
     /**
+     * Fetches the row count.
      *
-     * @param tableName
-     * @return
-     * @throws org.sleuthkit.autopsy.coreutils.SQLiteTableReaderException
+     * @param tableName Source table to count
+     *
+     * @return Count as an integer
+     *
+     * @throws SQLiteTableReaderException
      */
     public int getRowCount(String tableName) throws SQLiteTableReaderException {
         ensureOpen();
-        
         try (ResultSet countResult = conn.createStatement()
-                .executeQuery("SELECT count (*) as count FROM " + 
-                        "\"" + tableName + "\"")) {
+                .executeQuery("SELECT count (*) as count FROM "
+                        + "\"" + tableName + "\"")) {
             return countResult.getInt("count");
         } catch (SQLException ex) {
             throw new SQLiteTableReaderException(ex);
         }
     }
 
+    /**
+     * Fetches the column count of the table.
+     *
+     * @param tableName Source table to count
+     *
+     * @return Count as an integer
+     *
+     * @throws SQLiteTableReaderException
+     */
     public int getColumnCount(String tableName) throws SQLiteTableReaderException {
         ensureOpen();
-        
         try (ResultSet columnCount = conn.createStatement()
-                .executeQuery("SELECT * FROM " + 
-                        "\"" + tableName + "\"")) {
+                .executeQuery("SELECT * FROM "
+                        + "\"" + tableName + "\"")) {
             return columnCount.getMetaData().getColumnCount();
         } catch (SQLException ex) {
             throw new SQLiteTableReaderException(ex);
@@ -276,151 +314,135 @@ public class SQLiteTableReader implements AutoCloseable {
     }
 
     /**
+     * Reads column names and values from the table. Only actions that were
+     * configured in the Builder will be invoked during iteration. Iteration
+     * will stop when the table read has completed or an exception was
+     * encountered.
      *
-     * @param tableName
-     * @throws org.sleuthkit.autopsy.coreutils.SQLiteTableReaderException
+     * @param tableName Source table to read
+     *
+     * @throws SQLiteTableReaderException
      */
     public void read(String tableName) throws SQLiteTableReaderException {
-        readHelper("SELECT * FROM \"" + tableName +"\"", alwaysFalseCondition);
+        readHelper("SELECT * FROM \"" + tableName + "\"", () -> false);
     }
 
     /**
-     * Read x number of rows (limit), starting from row number y (offset) in
-     * table z (tableName).
+     * Reads column names and values from the table. Only actions that were
+     * configured in the Builder will be invoked during iteration. Column names
+     * are only read during the first call to this function. Iteration will stop
+     * when the table read has completed or an exception was encountered.
      *
-     * @param tableName
-     * @param limit
-     * @param offset
-     * @throws org.sleuthkit.autopsy.coreutils.SQLiteTableReaderException
+     * @param tableName Source table to perform a read
+     * @param limit Number of rows to read from the table
+     * @param offset Starting row to read from in the table
+     *
+     * @throws SQLiteTableReaderException
      *
      */
     public void read(String tableName, int limit, int offset) throws SQLiteTableReaderException {
-        readHelper("SELECT * FROM \"" + tableName +"\" LIMIT " + limit
-                + " OFFSET " + offset, alwaysFalseCondition);
+        readHelper("SELECT * FROM \"" + tableName + "\" LIMIT " + limit
+                + " OFFSET " + offset, () -> false);
     }
 
     /**
-     * Iterate through the table stopping if we are done, an exception is
-     * thrown, or the condition is false!
+     * Reads column names and values from the table. Iteration will stop when
+     * the condition is true.
      *
-     * @param tableName
-     * @param condition
-     * @throws org.sleuthkit.autopsy.coreutils.SQLiteTableReaderException
+     * @param tableName Source table to perform a read
+     * @param condition Condition to stop iteration when true
+     *
+     * @throws SQLiteTableReaderException
      *
      */
     public void read(String tableName, BooleanSupplier condition) throws SQLiteTableReaderException {
-        if(Objects.nonNull(prevTableName) && prevTableName.equals(tableName)) {
+        if (Objects.nonNull(prevTableName) && prevTableName.equals(tableName)) {
             readHelper("SELECT * FROM \"" + tableName + "\"", condition);
         } else {
             prevTableName = tableName;
-            closeResultSet();
+            closeTableResources();
             readHelper("SELECT * FROM \"" + tableName + "\"", condition);
         }
     }
 
     /**
-     * Iterate through the entire table calling the correct function given the
-     * datatype. Only stop when there is nothing left to read or a SQLException
-     * is thrown.
+     * Performs the result set iteration and is responsible for maintaining state 
+     * of the read over multiple invocations. 
      *
-     * @param tableName
-     *
-     * @throws org.sleuthkit.autopsy.core.AutopsySQLiteException
+     * @throws SQLiteTableReaderException
      */
     private void readHelper(String query, BooleanSupplier condition) throws SQLiteTableReaderException {
         try {
-            if(!hasOpened) {
-                openResultSet(query);
-                currentMetadata = queryResults.getMetaData();
-                currentColumnCount = currentMetadata.getColumnCount();
+            if (!liveResultSet) {
+                openTableResources(query);
                 columnNameIndex = 1;
             }
 
-            isFinished = false;
-            
-            for(; columnNameIndex <= currentColumnCount; columnNameIndex++) {
-                this.onColumnNameAction.accept(currentMetadata.getColumnName(columnNameIndex));
+            //Process column names before reading the database table values
+            for (; columnNameIndex <= currentColumnCount; columnNameIndex++) {
+                if (condition.getAsBoolean()) {
+                        return;
+                }
+                this.onColumnNameAction.accept(currentMetadata
+                        .getColumnName(columnNameIndex));
             }
 
-            while (unfinishedRowState || queryResults.next()) {
-                if (!unfinishedRowState) {
-                    currRowColumnIndex = 1;
-                }
-                
-                for (; currRowColumnIndex <= currentColumnCount; currRowColumnIndex++) {
-                    
+            //currRowColumnIndex > 0 means we are still reading the current result set row
+            while (currRowColumnIndex > 0 || queryResults.next()) {
+                while(currRowColumnIndex < currentColumnCount) {
                     if (condition.getAsBoolean()) {
-                        unfinishedRowState = true;
                         return;
                     }
                     
-                    //getObject automatically instiantiates the correct java data type
-                    Object item = queryResults.getObject(currRowColumnIndex);
-                    if(item instanceof String) {
+                    Object item = queryResults.getObject(++currRowColumnIndex);
+                    if (item instanceof String) {
                         this.onStringAction.accept((String) item);
-                    } else if(item instanceof Integer) {
+                    } else if (item instanceof Integer) {
                         this.onIntegerAction.accept((Integer) item);
-                    } else if(item instanceof Double) {
+                    } else if (item instanceof Double) {
                         this.onFloatAction.accept((Double) item);
-                    } else if(item instanceof Long) {
+                    } else if (item instanceof Long) {
                         this.onLongAction.accept((Long) item);
-                    } else if(item instanceof byte[]) {
+                    } else if (item instanceof byte[]) {
                         this.onBlobAction.accept((byte[]) item);
                     }
 
                     this.forAllAction.accept(item);
                 }
-                
-                unfinishedRowState = false;
+                //Wrap column index back around if we've reached the end of the row
+                currRowColumnIndex = (currRowColumnIndex % currentColumnCount);
             }
-            
-            isFinished = true;
-            closeResultSet();
+            closeTableResources();
         } catch (SQLException ex) {
-            closeResultSet();
-            isFinished = true;
+            closeTableResources();
             throw new SQLiteTableReaderException(ex);
         }
     }
 
     /**
+     * Ensures that the underlying database connection is open. This entails
+     * copying the abstract file contents to temp directory, copying over any
+     * WAL or SHM files and getting the connection from the DriverManager.
      *
-     * @throws org.sleuthkit.autopsy.core.AutopsySQLiteException
+     * @throws SQLiteTableReaderException
      */
     private void ensureOpen() throws SQLiteTableReaderException {
         if (Objects.isNull(conn)) {
             try {
                 Class.forName("org.sqlite.JDBC"); //NON-NLS  
-                String localDiskPath = writeAbstractFileToLocalDisk(file, file.getId());
-                findAndCopySQLiteMetaFile(file);
+                String localDiskPath = copyFileToTempDirectory(file, file.getId());
+
+                //Find and copy both WAL and SHM meta files 
+                findAndCopySQLiteMetaFile(file, file.getName() + "-wal");
+                findAndCopySQLiteMetaFile(file, file.getName() + "-shm");
                 conn = DriverManager.getConnection("jdbc:sqlite:" + localDiskPath);
-            } catch (NoCurrentCaseException | TskCoreException | IOException | 
-                    ClassNotFoundException | SQLException ex) {
+            } catch (NoCurrentCaseException | TskCoreException | IOException
+                    | ClassNotFoundException | SQLException ex) {
                 throw new SQLiteTableReaderException(ex);
             }
         }
     }
 
-    /**
-     * Overloaded implementation of
-     * {@link #findAndCopySQLiteMetaFile(AbstractFile, String) findAndCopySQLiteMetaFile}
-     * , automatically tries to copy -wal and -shm files without needing to know
-     * their existence.
-     *
-     * @param sqliteFile file which has -wal and -shm meta files
-     *
-     * @throws NoCurrentCaseException Case has been closed.
-     * @throws TskCoreException       fileManager cannot find AbstractFile
-     *                                files.
-     * @throws IOException            Issue during writing to file.
-     */
-    private void findAndCopySQLiteMetaFile(AbstractFile sqliteFile)
-            throws NoCurrentCaseException, TskCoreException, IOException {
-
-        findAndCopySQLiteMetaFile(sqliteFile, sqliteFile.getName() + "-wal");
-        findAndCopySQLiteMetaFile(sqliteFile, sqliteFile.getName() + "-shm");
-    }
-    
     /**
      * Searches for a meta file associated with the give SQLite database. If
      * found, it copies this file into the temp directory of the current case.
@@ -447,7 +469,7 @@ public class SQLiteTableReader implements AutoCloseable {
 
         if (metaFiles != null) {
             for (AbstractFile metaFile : metaFiles) {
-                writeAbstractFileToLocalDisk(metaFile, sqliteFile.getId());
+                copyFileToTempDirectory(metaFile, sqliteFile.getId());
             }
         }
     }
@@ -463,7 +485,7 @@ public class SQLiteTableReader implements AutoCloseable {
      * @throws IOException            Exception writing file contents
      * @throws NoCurrentCaseException Current case closed during file copying
      */
-    private String writeAbstractFileToLocalDisk(AbstractFile file, long id)
+    private String copyFileToTempDirectory(AbstractFile file, long id)
             throws IOException, NoCurrentCaseException {
 
         String localDiskPath = Case.getCurrentCaseThrows().getTempDirectory()
@@ -476,72 +498,80 @@ public class SQLiteTableReader implements AutoCloseable {
     }
 
     /**
+     * Executes the query and assigns resource references to instance variables.
      *
-     * @param query
-     * @throws SQLException 
+     * @param query Input query to execute
+     *
+     * @throws SQLiteTableReaderException
      */
-    private void openResultSet(String query) throws SQLiteTableReaderException {
-        ensureOpen();
-        
+    private void openTableResources(String query) throws SQLiteTableReaderException {
         try {
+            ensureOpen();
             statement = conn.prepareStatement(query);
-            
             queryResults = statement.executeQuery();
-            hasOpened = true;
+            currentMetadata = queryResults.getMetaData();
+            currentColumnCount = currentMetadata.getColumnCount();
+            liveResultSet = true;
         } catch (SQLException ex) {
             throw new SQLiteTableReaderException(ex);
         }
     }
 
     /**
-     * 
+     * Ensures both the statement and the result set for a table are closed.
      */
-    private void closeResultSet() {
+    private void closeTableResources() {
         try {
-            if(Objects.nonNull(statement)) {
+            if (Objects.nonNull(statement)) {
                 statement.close();
             }
-            if(Objects.nonNull(queryResults)) {
+            if (Objects.nonNull(queryResults)) {
                 queryResults.close();
             }
-            hasOpened = false;
+            liveResultSet = false;
         } catch (SQLException ex) {
             //Do nothing, can't close.. tried our best.
         }
     }
 
     /**
-     * Closes all connections with the database.
+     * Closes all resources attached to the database file.
+     *
+     * @throws SQLiteTableReaderException
      */
     @Override
-    public void close() {
+    public void close() throws SQLiteTableReaderException {
         try {
-            closeResultSet();
-            if(Objects.nonNull(conn)) {
+            closeTableResources();
+            if (Objects.nonNull(conn)) {
                 conn.close();
             }
         } catch (SQLException ex) {
-            //Do nothing, can't close.. tried our best.
+            throw new SQLiteTableReaderException(ex);
         }
     }
 
     /**
-     * Checks if there is still work to do on the result set.
+     * Provides status of the current read operation.
      *
-     * @return boolean
+     * @return
      */
     public boolean isFinished() {
-        return isFinished;
+        return !liveResultSet;
     }
 
     /**
-     * Last ditch effort to close the connections.
+     * Last ditch effort to close the connections during garbage collection.
      *
      * @throws Throwable
      */
     @Override
     public void finalize() throws Throwable {
         super.finalize();
-        close();
+        try {
+            close();
+        } catch (SQLiteTableReaderException ex) {
+            //Do nothing, we tried out best to close the connection.
+        }
     }
 }

Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReaderException.java

@@ -1,20 +1,43 @@
 /*
- * To change this license header, choose License Headers in Project Properties.
- * To change this template file, choose Tools | Templates
- * and open the template in the editor.
+ * Autopsy Forensic Browser
+ *
+ * Copyright 2018-2018 Basis Technology Corp.
+ * Contact: carrier <at> sleuthkit <dot> org
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
 package org.sleuthkit.autopsy.coreutils;
 
 /**
- *
- * @author dsmyda
+ * Provides a system exception for the SQLiteTableReader class. 
  */
 public class SQLiteTableReaderException extends Exception {
     
+    /**
+     * Accepts both a message and a parent exception.
+     * 
+     * @param msg Message detailing the cause
+     * @param ex Parent exception
+     */
     public SQLiteTableReaderException(String msg, Throwable ex) {
         super(msg, ex);
     }
     
+    /**
+     * Accepts only a parent exception.
+     * 
+     * @param ex Parent exception
+     */
     public SQLiteTableReaderException(Throwable ex) {
         super(ex);
     }