From eeace0490415ebb23106757fbd77944882edb1ca Mon Sep 17 00:00:00 2001 From: Raman Date: Tue, 31 Jul 2018 11:44:11 -0400 Subject: [PATCH 1/2] 1003: Add transactions support in CaseDbAccessManager api. --- .../imagegallery/ImageGalleryController.java | 15 +++-- .../imagegallery/datamodel/DrawableDB.java | 56 +++++++++++++------ 2 files changed, 49 insertions(+), 22 deletions(-) diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java index 76c3626512..53544f891a 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java @@ -88,6 +88,7 @@ import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.DataSource; import org.sleuthkit.datamodel.SleuthkitCase; +import org.sleuthkit.datamodel.SleuthkitCase.CaseDbTransaction; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskData; @@ -769,7 +770,7 @@ public final class ImageGalleryController { abstract List getFiles() throws TskCoreException; - abstract void processFile(final AbstractFile f, DrawableDB.DrawableTransaction tr) throws TskCoreException; + abstract void processFile(final AbstractFile f, DrawableDB.DrawableTransaction tr, CaseDbTransaction caseDBTransaction) throws TskCoreException; @Override public void run() { @@ -788,6 +789,7 @@ public final class ImageGalleryController { //do in transaction DrawableDB.DrawableTransaction tr = taskDB.beginTransaction(); + CaseDbTransaction caseDbTransaction = tskCase.beginTransaction(); int workDone = 0; for (final AbstractFile f : files) { if (isCancelled()) { @@ -800,7 +802,7 @@ public final class ImageGalleryController { LOGGER.log(Level.WARNING, "BulkTransferTask interrupted. Ignoring it to update the contents of drawable database."); //NON-NLS } - processFile(f, tr); + processFile(f, tr, caseDbTransaction); workDone++; progressHandle.progress(f.getName(), workDone); @@ -815,6 +817,7 @@ public final class ImageGalleryController { progressHandle.start(); taskDB.commitTransaction(tr, true); + caseDbTransaction.commit(); } catch (TskCoreException ex) { progressHandle.progress(Bundle.BulkTask_stopCopy_status()); @@ -865,7 +868,7 @@ public final class ImageGalleryController { } @Override - void processFile(AbstractFile f, DrawableDB.DrawableTransaction tr) throws TskCoreException { + void processFile(AbstractFile f, DrawableDB.DrawableTransaction tr, CaseDbTransaction caseDbTransaction) throws TskCoreException { final boolean known = f.getKnown() == TskData.FileKnown.KNOWN; if (known) { @@ -875,7 +878,7 @@ public final class ImageGalleryController { try { //supported mimetype => analyzed if ( null != f.getMIMEType() && FileTypeUtils.hasDrawableMIMEType(f)) { - taskDB.updateFile(DrawableFile.create(f, true, false), tr); + taskDB.updateFile(DrawableFile.create(f, true, false), tr, caseDbTransaction ); } else { //unsupported mimtype => analyzed but shouldn't include @@ -927,8 +930,8 @@ public final class ImageGalleryController { } @Override - void processFile(final AbstractFile f, DrawableDB.DrawableTransaction tr) { - taskDB.insertFile(DrawableFile.create(f, false, false), tr); + void processFile(final AbstractFile f, DrawableDB.DrawableTransaction tr, CaseDbTransaction caseDBTransaction) { + taskDB.insertFile(DrawableFile.create(f, false, false), tr, caseDBTransaction); } @Override diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java index a578641c22..3f913a5221 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java @@ -63,6 +63,7 @@ import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.ContentTag; import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.CaseDbAccessManager.CaseDbAccessQueryCallback; +import org.sleuthkit.datamodel.SleuthkitCase.CaseDbTransaction; import org.sleuthkit.datamodel.TagName; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskData.DbType; @@ -230,9 +231,16 @@ public final class DrawableDB { insertHashHitStmt = prepareStatement("INSERT OR IGNORE INTO hash_set_hits (hash_set_id, obj_id) VALUES (?,?)"); //NON-NLS - for (DhsImageCategory cat : DhsImageCategory.values()) { - insertGroup(cat.getDisplayName(), DrawableAttribute.CATEGORY); + try { + CaseDbTransaction caseDbTransaction = tskCase.beginTransaction(); + for (DhsImageCategory cat : DhsImageCategory.values()) { + insertGroup(cat.getDisplayName(), DrawableAttribute.CATEGORY, caseDbTransaction); + } + caseDbTransaction.commit(); + } catch (TskCoreException ex) { + throw new ExceptionInInitializerError(ex); } + initializeImageList(); } else { throw new ExceptionInInitializerError(); @@ -599,23 +607,38 @@ public final class DrawableDB { } public void updateFile(DrawableFile f) { - DrawableTransaction trans = beginTransaction(); - updateFile(f, trans); - commitTransaction(trans, true); + try { + DrawableTransaction trans = beginTransaction(); + CaseDbTransaction caseDbTransaction = tskCase.beginTransaction(); + updateFile(f, trans, caseDbTransaction); + commitTransaction(trans, true); + caseDbTransaction.commit(); + } + catch (TskCoreException ex) { + LOGGER.log(Level.SEVERE, "Error updating file", ex); //NON-NLS + } + } public void insertFile(DrawableFile f) { - DrawableTransaction trans = beginTransaction(); - insertFile(f, trans); - commitTransaction(trans, true); + try { + DrawableTransaction trans = beginTransaction(); + CaseDbTransaction caseDbTransaction = this.tskCase.beginTransaction(); + insertFile(f, trans, caseDbTransaction); + commitTransaction(trans, true); + caseDbTransaction.commit(); + } + catch (TskCoreException ex) { + LOGGER.log(Level.SEVERE, "Error inserting file", ex); //NON-NLS + } } - public void insertFile(DrawableFile f, DrawableTransaction tr) { - insertOrUpdateFile(f, tr, insertFileStmt); + public void insertFile(DrawableFile f, DrawableTransaction tr, CaseDbTransaction caseDbTransaction) { + insertOrUpdateFile(f, tr, insertFileStmt, caseDbTransaction); } - public void updateFile(DrawableFile f, DrawableTransaction tr) { - insertOrUpdateFile(f, tr, updateFileStmt); + public void updateFile(DrawableFile f, DrawableTransaction tr, CaseDbTransaction caseDbTransaction) { + insertOrUpdateFile(f, tr, updateFileStmt, caseDbTransaction); } /** @@ -631,7 +654,7 @@ public final class DrawableDB { * @param tr a transaction to use, must not be null * @param stmt the statement that does the actull inserting */ - private void insertOrUpdateFile(DrawableFile f, @Nonnull DrawableTransaction tr, @Nonnull PreparedStatement stmt) { + private void insertOrUpdateFile(DrawableFile f, @Nonnull DrawableTransaction tr, @Nonnull PreparedStatement stmt, @Nonnull CaseDbTransaction caseDbTransaction) { if (tr.isClosed()) { throw new IllegalArgumentException("can't update database with closed transaction"); @@ -683,7 +706,7 @@ public final class DrawableDB { for (Comparable val : vals) { //use empty string for null values (mime_type), this shouldn't happen! if (null != val) { - insertGroup(val.toString(), attr); + insertGroup(val.toString(), attr, caseDbTransaction); } } } @@ -1008,8 +1031,9 @@ public final class DrawableDB { * Insert new group into DB * @param value Value of the group (unique to the type) * @param groupBy Type of the grouping (CATEGORY, MAKE, etc.) + * @param caseDbTransaction transaction to use for CaseDB insert/updates */ - private void insertGroup(final String value, DrawableAttribute groupBy) { + private void insertGroup(final String value, DrawableAttribute groupBy, CaseDbTransaction caseDbTransaction) { String insertSQL = ""; try { insertSQL = String.format(" (value, attribute) VALUES (\'%s\', \'%s\')", value, groupBy.attrName.toString());; @@ -1018,7 +1042,7 @@ public final class DrawableDB { insertSQL += String.format(" ON CONFLICT (value, attribute) DO UPDATE SET value = \'%s\', attribute=\'%s\'", value, groupBy.attrName.toString()); } - tskCase.getCaseDbAccessManager().insertOrUpdate(GROUPS_TABLENAME, insertSQL); + tskCase.getCaseDbAccessManager().insertOrUpdate(GROUPS_TABLENAME, insertSQL, caseDbTransaction); } catch (TskCoreException ex) { // Don't need to report it if the case was closed if (Case.isCaseOpen()) { From a0c6d71f93bdb5e6791ff1147a1b9057ace7f795 Mon Sep 17 00:00:00 2001 From: Raman Date: Wed, 1 Aug 2018 12:57:07 -0400 Subject: [PATCH 2/2] 1003: Add transaction support to CaseDbAccessManager api. - address review comments in previous commit. --- .../imagegallery/ImageGalleryController.java | 50 ++++++++++------- .../imagegallery/datamodel/DrawableDB.java | 53 ++++++++++++++++--- 2 files changed, 78 insertions(+), 25 deletions(-) diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java index 53544f891a..fa172f0919 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java @@ -778,6 +778,9 @@ public final class ImageGalleryController { progressHandle.start(); updateMessage(Bundle.CopyAnalyzedFiles_populatingDb_status()); + + DrawableDB.DrawableTransaction drawableDbTransaction = null; + CaseDbTransaction caseDbTransaction = null; try { //grab all files with supported extension or detected mime types final List files = getFiles(); @@ -785,24 +788,23 @@ public final class ImageGalleryController { updateProgress(0.0); - taskCompletionStatus = true; - - //do in transaction - DrawableDB.DrawableTransaction tr = taskDB.beginTransaction(); - CaseDbTransaction caseDbTransaction = tskCase.beginTransaction(); + taskCompletionStatus = true; int workDone = 0; + + //do in transaction + drawableDbTransaction = taskDB.beginTransaction(); + caseDbTransaction = tskCase.beginTransaction(); for (final AbstractFile f : files) { - if (isCancelled()) { - LOGGER.log(Level.WARNING, "Task cancelled: not all contents may be transfered to drawable database."); //NON-NLS + if (isCancelled() || Thread.interrupted()) { + LOGGER.log(Level.WARNING, "Task cancelled or interrupted: not all contents may be transfered to drawable database."); //NON-NLS + taskCompletionStatus = false; progressHandle.finish(); + break; } - if (Thread.interrupted()) { - LOGGER.log(Level.WARNING, "BulkTransferTask interrupted. Ignoring it to update the contents of drawable database."); //NON-NLS - } - processFile(f, tr, caseDbTransaction); + processFile(f, drawableDbTransaction, caseDbTransaction); workDone++; progressHandle.progress(f.getName(), workDone); @@ -816,10 +818,20 @@ public final class ImageGalleryController { updateProgress(1.0); progressHandle.start(); - taskDB.commitTransaction(tr, true); + taskDB.commitTransaction(drawableDbTransaction, true); caseDbTransaction.commit(); - } catch (TskCoreException ex) { + } catch (TskCoreException ex) { + if (null != drawableDbTransaction) { + taskDB.rollbackTransaction(drawableDbTransaction); + } + if (null != caseDbTransaction) { + try { + caseDbTransaction.rollback(); + } catch (TskCoreException ex2) { + LOGGER.log(Level.SEVERE, "Error in trying to rollback transaction", ex2); //NON-NLS + } + } progressHandle.progress(Bundle.BulkTask_stopCopy_status()); LOGGER.log(Level.WARNING, "Stopping copy to drawable db task. Failed to transfer all database contents", ex); //NON-NLS MessageNotifyUtil.Notify.warn(Bundle.BulkTask_errPopulating_errMsg(), ex.getMessage()); @@ -880,13 +892,14 @@ public final class ImageGalleryController { if ( null != f.getMIMEType() && FileTypeUtils.hasDrawableMIMEType(f)) { taskDB.updateFile(DrawableFile.create(f, true, false), tr, caseDbTransaction ); } - else { //unsupported mimtype => analyzed but shouldn't include - + else { // if mimetype of the file hasn't been ascertained, ingest might not have completed yet. if (null == f.getMIMEType()) { this.setTaskCompletionStatus(false); + } else { + //unsupported mimtype => analyzed but shouldn't include + taskDB.removeFile(f.getId(), tr); } - taskDB.removeFile(f.getId(), tr); } } catch (FileTypeDetector.FileTypeDetectorInitException ex) { throw new TskCoreException("Failed to initialize FileTypeDetector.", ex); @@ -999,13 +1012,12 @@ public final class ImageGalleryController { } } } catch (TskCoreException | FileTypeDetector.FileTypeDetectorInitException ex) { - //TODO: What to do here? LOGGER.log(Level.SEVERE, "Unable to determine if file is drawable and not known. Not making any changes to DB", ex); //NON-NLS MessageNotifyUtil.Notify.error("Image Gallery Error", "Unable to determine if file is drawable and not known. Not making any changes to DB. See the logs for details."); } } - } else { //TODO: keep track of what we missed for later + } else { setStale(true); } } @@ -1044,7 +1056,7 @@ public final class ImageGalleryController { Content newDataSource = (Content) evt.getNewValue(); if (isListeningEnabled()) { queueDBTask(new PrePopulateDataSourceFiles(newDataSource, ImageGalleryController.this, getDatabase(), getSleuthKitCase())); - } else {//TODO: keep track of what we missed for later + } else { setStale(true); } } diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java index 3f913a5221..6221de8947 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java @@ -231,13 +231,22 @@ public final class DrawableDB { insertHashHitStmt = prepareStatement("INSERT OR IGNORE INTO hash_set_hits (hash_set_id, obj_id) VALUES (?,?)"); //NON-NLS + CaseDbTransaction caseDbTransaction = null; try { - CaseDbTransaction caseDbTransaction = tskCase.beginTransaction(); + caseDbTransaction = tskCase.beginTransaction(); for (DhsImageCategory cat : DhsImageCategory.values()) { insertGroup(cat.getDisplayName(), DrawableAttribute.CATEGORY, caseDbTransaction); } caseDbTransaction.commit(); } catch (TskCoreException ex) { + if (null != caseDbTransaction) { + try { + caseDbTransaction.rollback(); + } + catch(TskCoreException ex2) { + LOGGER.log(Level.SEVERE, "Error in trying to rollback transaction", ex2); + } + } throw new ExceptionInInitializerError(ex); } @@ -607,28 +616,53 @@ public final class DrawableDB { } public void updateFile(DrawableFile f) { - try { - DrawableTransaction trans = beginTransaction(); - CaseDbTransaction caseDbTransaction = tskCase.beginTransaction(); + DrawableTransaction trans = null; + CaseDbTransaction caseDbTransaction = null; + + try { + trans = beginTransaction(); + caseDbTransaction = tskCase.beginTransaction(); updateFile(f, trans, caseDbTransaction); commitTransaction(trans, true); caseDbTransaction.commit(); } catch (TskCoreException ex) { + if (null != trans) { + rollbackTransaction(trans); + } + if (null != caseDbTransaction) { + try { + caseDbTransaction.rollback(); + } catch (TskCoreException ex2) { + LOGGER.log(Level.SEVERE, "Error in trying to rollback transaction", ex2); //NON-NLS + } + } LOGGER.log(Level.SEVERE, "Error updating file", ex); //NON-NLS } } public void insertFile(DrawableFile f) { + DrawableTransaction trans = null; + CaseDbTransaction caseDbTransaction = null; try { - DrawableTransaction trans = beginTransaction(); - CaseDbTransaction caseDbTransaction = this.tskCase.beginTransaction(); + trans = beginTransaction(); + caseDbTransaction = tskCase.beginTransaction(); insertFile(f, trans, caseDbTransaction); commitTransaction(trans, true); caseDbTransaction.commit(); } catch (TskCoreException ex) { + if (null != trans) { + rollbackTransaction(trans); + } + if (null != caseDbTransaction) { + try { + caseDbTransaction.rollback(); + } catch (TskCoreException ex2) { + LOGGER.log(Level.SEVERE, "Error in trying to rollback transaction", ex2); //NON-NLS + } + } LOGGER.log(Level.SEVERE, "Error inserting file", ex); //NON-NLS } } @@ -799,6 +833,13 @@ public final class DrawableDB { tr.commit(notify); } + public void rollbackTransaction(DrawableTransaction tr) { + if (tr.isClosed()) { + throw new IllegalArgumentException("can't rollback already closed transaction"); + } + tr.rollback(); + } + public Boolean isFileAnalyzed(DrawableFile f) { return isFileAnalyzed(f.getId()); }