diff --git a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/DocumentEmbeddedContentExtractor.java b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/DocumentEmbeddedContentExtractor.java index a362c5789f..35f27ca8f6 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/DocumentEmbeddedContentExtractor.java +++ b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/DocumentEmbeddedContentExtractor.java @@ -22,6 +22,9 @@ import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.nio.file.InvalidPathException; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; @@ -57,6 +60,7 @@ import org.openide.util.NbBundle; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.autopsy.casemodule.services.FileManager; +import static org.sleuthkit.autopsy.coreutils.FileUtil.escapeFileName; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.ingest.IngestJobContext; import org.sleuthkit.autopsy.ingest.IngestServices; @@ -159,7 +163,8 @@ class DocumentEmbeddedContentExtractor { void extractEmbeddedContent(AbstractFile abstractFile) { List listOfExtractedImages = null; List listOfExtractedImageAbstractFiles = null; - this.parentFileName = EmbeddedFileExtractorIngestModule.getUniqueName(abstractFile); + //save the parent file name with out illegal windows characters + this.parentFileName = utf8SanitizeFileName(EmbeddedFileExtractorIngestModule.getUniqueName(abstractFile)); // Skip files that already have been unpacked. try { @@ -289,7 +294,6 @@ class DocumentEmbeddedContentExtractor { // These get thrown in certain images. The reason is unknown. It is // likely due to problems with the file formats that POI is poorly // handling. - //Any runtime exception escaping LOGGER.log(Level.WARNING, "Word document container could not be initialized. Reason: {0}", ex.getMessage()); //NON-NLS return null; @@ -308,7 +312,7 @@ class DocumentEmbeddedContentExtractor { byte[] data = null; int pictureNumber = 0; //added to ensure uniqueness in cases where suggestFullFileName returns duplicates for (Picture picture : listOfAllPictures) { - String fileName = UNKNOWN_IMAGE_NAME_PREFIX +pictureNumber +"."+ picture.suggestFileExtension(); + String fileName = UNKNOWN_IMAGE_NAME_PREFIX + pictureNumber + "." + picture.suggestFileExtension(); try { data = picture.getContent(); } catch (Exception ex) { @@ -474,11 +478,12 @@ class DocumentEmbeddedContentExtractor { return listOfExtractedImages; } - + /** * Extracts embedded attachments from PDF files. - * + * * @param abstractFile Input PDF file + * * @return List of extracted files to be made into derived file instances. */ private List extractEmbeddedContentFromPDF(AbstractFile abstractFile) { @@ -489,20 +494,20 @@ class DocumentEmbeddedContentExtractor { Map extractedAttachments = pdfExtractor.extract( new ReadContentInputStream(abstractFile), abstractFile.getId(), outputDirectory); - + //Convert output to hook into the existing logic for creating derived files List extractedFiles = new ArrayList<>(); extractedAttachments.entrySet().forEach((pathEntry) -> { String fileName = pathEntry.getKey(); Path writeLocation = pathEntry.getValue(); extractedFiles.add(new ExtractedFile(fileName, - getFileRelativePath(writeLocation.getFileName().toString()), + getFileRelativePath(writeLocation.getFileName().toString()), writeLocation.toFile().length())); }); - + return extractedFiles; - } catch (IOException | SAXException | TikaException ex) { - LOGGER.log(Level.WARNING, "Error attempting to extract attachments from PDFs", ex); //NON-NLS + } catch (IOException | SAXException | TikaException | InvalidPathException ex) { + LOGGER.log(Level.WARNING, "Error attempting to extract attachments from PDFs for file Name: " + abstractFile.getName() + " ID: " + abstractFile.getId(), ex); //NON-NLS } return Collections.emptyList(); } @@ -557,6 +562,19 @@ class DocumentEmbeddedContentExtractor { return Paths.get(moduleDirRelative, this.parentFileName, fileName).toString(); } + /** + * UTF-8 sanitize and escape special characters in a file name or a file + * name component + * + * @param fileName to escape + * + * @return Sanitized string + */ + private static String utf8SanitizeFileName(String fileName) { + Charset charset = StandardCharsets.UTF_8; + return charset.decode(charset.encode(escapeFileName(fileName))).toString(); + } + /** * Represents a file extracted using either Tika or POI methods. Currently, * POI is not capable of extracting ctime, crtime, mtime, and atime; these @@ -669,6 +687,8 @@ class DocumentEmbeddedContentExtractor { //that might be included in the name) and make sure //to normalize the name name = FilenameUtils.normalize(FilenameUtils.getName(name)); + //remove any illegal characters from name + name = utf8SanitizeFileName(name); } // Get the suggested extension based on mime type. diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Chunker.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Chunker.java index 82494f2f0d..08ca0ab511 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Chunker.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Chunker.java @@ -45,33 +45,69 @@ class Chunker implements Iterator, Iterable { private static final Charset UTF_8 = StandardCharsets.UTF_8; //Chunking algorithm paramaters-------------------------------------// - /** the maximum size of a chunk, including the window. */ + /** + * the maximum size of a chunk, including the window. + */ private static final int MAX_TOTAL_CHUNK_SIZE = 32760; //bytes - /** the minimum to read before we start the process of looking for - * whitespace to break at and creating an overlapping window. */ + /** + * the minimum to read before we start the process of looking for whitespace + * to break at and creating an overlapping window. + */ private static final int MINIMUM_BASE_CHUNK_SIZE = 30 * 1024; //bytes - /** The maximum size of the chunk, before the overlapping window, even if we - * couldn't find whitespace to break at. */ + /** + * The maximum size of the chunk, before the overlapping window, even if we + * couldn't find whitespace to break at. + */ private static final int MAXIMUM_BASE_CHUNK_SIZE = 31 * 1024; //bytes - /** The amount of text we will read through before we give up on finding - * whitespace to break the chunk/window at. */ + /** + * The amount of text we will read through before we give up on finding + * whitespace to break the chunk/window at. + */ private static final int WHITE_SPACE_BUFFER_SIZE = 512; //bytes - /** The number of characters to read in one go from the Reader. */ + /** + * The number of characters to read in one go from the Reader. + */ private static final int READ_CHARS_BUFFER_SIZE = 512; //chars + /** + * When toLowerCase() is called on a character, the lower cased output + * can be different in size than the original input. I have seen a single + * input character turn into 3 characters (and 5 bytes) after lowercasing. + * I could not find any info as to what is the upper limit of how much a + * character can "increase in size" during lower casing. I'm guestimating + * and setting that limit at 10 bytes. + */ + private static final int MAX_CHAR_SIZE_INCREASE_IN_BYTES = 10; //bytes ////chunker state--------------------------------------------/// - /** The Reader that this chunk reads from, and divides into chunks. It must - * be a buffered reader to ensure that mark/reset are supported. */ + /** + * The Reader that this chunk reads from, and divides into chunks. It must + * be a buffered reader to ensure that mark/reset are supported. + */ private final PushbackReader reader; - /** The local buffer of characters read from the Reader. */ + /** + * The local buffer of characters read from the Reader. + */ private final char[] tempChunkBuf = new char[READ_CHARS_BUFFER_SIZE]; - /** the size in bytes of the chunk (so far). */ + /** + * the size in bytes of the chunk (so far). + */ private int chunkSizeBytes = 0; - /** Has the chunker reached the end of the Reader? If so, there are no more - * chunks, and the current chunk does not need a window. */ + + /** + * the size in bytes of the lowercased chunk (so far). Note that lowercasing + * in Java can change the size of the string so we need to make sure the + * lowercased string also fits in MAX_TOTAL_CHUNK_SIZE. + */ + private int lowerCasedChunkSizeBytes = 0; + /** + * Has the chunker reached the end of the Reader? If so, there are no more + * chunks, and the current chunk does not need a window. + */ private boolean endOfReaderReached = false; - /** Store any exception encountered reading from the Reader. */ + /** + * Store any exception encountered reading from the Reader. + */ private Exception ex; /** @@ -140,7 +176,7 @@ class Chunker implements Iterator, Iterable { * @param s The string to cleanup. * * @return A StringBuilder with the same content as s but where all invalid - * code * points have been replaced. + * code * points have been replaced. */ private static StringBuilder replaceInvalidUTF16(String s) { /* encode the string to UTF-16 which does the replcement, see @@ -162,16 +198,18 @@ class Chunker implements Iterator, Iterable { //reset state for the next chunk chunkSizeBytes = 0; + lowerCasedChunkSizeBytes = 0; int baseChunkSizeChars = 0; StringBuilder currentChunk = new StringBuilder(); StringBuilder currentWindow = new StringBuilder(); + StringBuilder lowerCasedChunk = new StringBuilder(); try { - currentChunk.append(readBaseChunk()); + readBaseChunk(currentChunk, lowerCasedChunk); baseChunkSizeChars = currentChunk.length(); //save the base chunk length - currentWindow.append(readWindow()); - //add the window text to the current chunk. - currentChunk.append(currentWindow); + readWindow(currentWindow, lowerCasedChunk); + //add the window text to the current chunk. + currentChunk.append(currentWindow); if (endOfReaderReached) { /* if we have reached the end of the content,we won't make * another overlapping chunk, so the length of the base chunk @@ -186,9 +224,9 @@ class Chunker implements Iterator, Iterable { * and break any chunking loop in client code. */ ex = ioEx; } - + //sanitize the text and return a Chunk object, that includes the base chunk length. - return new Chunk(currentChunk, baseChunkSizeChars, chunkSizeBytes); + return new Chunk(currentChunk, baseChunkSizeChars, lowerCasedChunk); } /** @@ -196,14 +234,12 @@ class Chunker implements Iterator, Iterable { * * @throws IOException if there is a problem reading from the reader. */ - private StringBuilder readBaseChunk() throws IOException { - StringBuilder currentChunk = new StringBuilder(); + private void readBaseChunk(StringBuilder currentChunk, StringBuilder lowerCasedChunk) throws IOException { //read the chunk until the minimum base chunk size - readHelper(MINIMUM_BASE_CHUNK_SIZE, currentChunk); + readHelper(MINIMUM_BASE_CHUNK_SIZE, currentChunk, lowerCasedChunk); //keep reading until the maximum base chunk size or white space is reached. - readToWhiteSpaceHelper(MAXIMUM_BASE_CHUNK_SIZE, currentChunk); - return currentChunk; + readToWhiteSpaceHelper(MAXIMUM_BASE_CHUNK_SIZE, currentChunk, lowerCasedChunk); } /** @@ -211,14 +247,12 @@ class Chunker implements Iterator, Iterable { * * @throws IOException if there is a problem reading from the reader. */ - private StringBuilder readWindow() throws IOException { - StringBuilder currentWindow = new StringBuilder(); + private void readWindow(StringBuilder currentChunk, StringBuilder lowerCasedChunk) throws IOException { //read the window, leaving some room to look for white space to break at. - readHelper(MAX_TOTAL_CHUNK_SIZE - WHITE_SPACE_BUFFER_SIZE, currentWindow); + readHelper(MAX_TOTAL_CHUNK_SIZE - WHITE_SPACE_BUFFER_SIZE, currentChunk, lowerCasedChunk); //keep reading until the max chunk size, or until whitespace is reached. - readToWhiteSpaceHelper(MAX_TOTAL_CHUNK_SIZE, currentWindow); - return currentWindow; + readToWhiteSpaceHelper(MAX_TOTAL_CHUNK_SIZE, currentChunk, lowerCasedChunk); } /** @@ -229,10 +263,10 @@ class Chunker implements Iterator, Iterable { * * @throws IOException */ - private void readHelper(int maxBytes, StringBuilder currentSegment) throws IOException { + private void readHelper(int maxBytes, StringBuilder currentSegment, StringBuilder currentLowerCasedSegment) throws IOException { int charsRead = 0; //read chars up to maxBytes, or the end of the reader. - while ((chunkSizeBytes < maxBytes) + while ((chunkSizeBytes < maxBytes) && (lowerCasedChunkSizeBytes < maxBytes) && (endOfReaderReached == false)) { charsRead = reader.read(tempChunkBuf, 0, READ_CHARS_BUFFER_SIZE); if (-1 == charsRead) { @@ -253,11 +287,19 @@ class Chunker implements Iterator, Iterable { //get the length in utf8 bytes of the read chars int segmentSize = chunkSegment.toString().getBytes(UTF_8).length; + // lower case the string and get it's size. NOTE: lower casing can + // change the size of the string! + String lowerCasedSegment = chunkSegment.toString().toLowerCase(); + int lowerCasedSegmentSize = lowerCasedSegment.getBytes(UTF_8).length; + //if it will not put us past maxBytes - if (chunkSizeBytes + segmentSize < maxBytes) { + if ((chunkSizeBytes + segmentSize < maxBytes) && (lowerCasedChunkSizeBytes + lowerCasedSegmentSize < maxBytes)) { //add it to the chunk currentSegment.append(chunkSegment); chunkSizeBytes += segmentSize; + + currentLowerCasedSegment.append(lowerCasedSegment); + lowerCasedChunkSizeBytes += lowerCasedSegmentSize; } else { //unread it, and break out of read loop. reader.unread(tempChunkBuf, 0, charsRead); @@ -275,11 +317,12 @@ class Chunker implements Iterator, Iterable { * * @throws IOException */ - private void readToWhiteSpaceHelper(int maxBytes, StringBuilder currentChunk) throws IOException { + private void readToWhiteSpaceHelper(int maxBytes, StringBuilder currentChunk, StringBuilder lowerCasedChunk) throws IOException { int charsRead = 0; boolean whitespaceFound = false; //read 1 char at a time up to maxBytes, whitespaceFound, or we reach the end of the reader. - while ((chunkSizeBytes < maxBytes) + while ((chunkSizeBytes < maxBytes - MAX_CHAR_SIZE_INCREASE_IN_BYTES) + && (lowerCasedChunkSizeBytes < maxBytes - MAX_CHAR_SIZE_INCREASE_IN_BYTES) && (whitespaceFound == false) && (endOfReaderReached == false)) { charsRead = reader.read(tempChunkBuf, 0, 1); @@ -314,6 +357,12 @@ class Chunker implements Iterator, Iterable { //add read chars to the chunk and update the length. currentChunk.append(sanitizedChunkSegment); chunkSizeBytes += sanitizedChunkSegment.toString().getBytes(UTF_8).length; + + // lower case the string and get it's size. NOTE: lower casing can + // change the size of the string. + String lowerCasedSegment = sanitizedChunkSegment.toString().toLowerCase(); + lowerCasedChunk.append(lowerCasedSegment); + lowerCasedChunkSizeBytes += lowerCasedSegment.getBytes(UTF_8).length; } } } @@ -326,16 +375,16 @@ class Chunker implements Iterator, Iterable { private final StringBuilder sb; private final int baseChunkSizeChars; - private final int chunkSizeBytes; + private final StringBuilder lowerCasedChunk; - Chunk(StringBuilder sb, int baseChunkSizeChars, int chunkSizeBytes) { + Chunk(StringBuilder sb, int baseChunkSizeChars, StringBuilder lowerCasedChunk) { this.sb = sb; this.baseChunkSizeChars = baseChunkSizeChars; - this.chunkSizeBytes = chunkSizeBytes; + this.lowerCasedChunk = lowerCasedChunk; } /** - * Get the content of the chunk. + * Get the content of the original (non-lower cased) chunk. * * @return The content of the chunk. */ @@ -345,16 +394,16 @@ class Chunker implements Iterator, Iterable { } /** - * Get the size in bytes of the utf-8 encoding of the entire chunk. + * Get the content of the lower cased chunk. * - * @return the size in bytes of the utf-8 encoding of the entire chunk + * @return The content of the chunk. */ - public int getChunkSizeBytes() { - return chunkSizeBytes; + public String geLowerCasedChunk() { + return lowerCasedChunk.toString(); } /** - * Get the length of the base chunk in java chars. + * Get the length of the original (non-lower cased) base chunk in java chars. * * @return the length of the base chunk in java chars. */ diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/HighlightedText.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/HighlightedText.java index 99d4f40820..1eb30f5b6c 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/HighlightedText.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/HighlightedText.java @@ -391,7 +391,7 @@ class HighlightedText implements IndexedText { } //tune the highlighter - if (shouldUseOriginalHighlighter(contentIdStr)) { + if (shouldUseOriginalHighlighter(filterQuery)) { // use original highlighter q.setParam("hl.useFastVectorHighlighter", "off"); q.setParam("hl.simple.pre", HIGHLIGHT_PRE); @@ -618,11 +618,13 @@ class HighlightedText implements IndexedText { * > 降っています * Unified highlighter (from Solr 6.4) handles the case as expected: * > 雨が降っています。 + * + * @param filterQuery An already properly escaped filter query. */ - private boolean shouldUseOriginalHighlighter(String contentID) throws NoOpenCoreException, KeywordSearchModuleException { + private boolean shouldUseOriginalHighlighter(String filterQuery) throws NoOpenCoreException, KeywordSearchModuleException { final SolrQuery q = new SolrQuery(); q.setQuery("*:*"); - q.addFilterQuery(Server.Schema.ID.toString() + ":" + contentID); + q.addFilterQuery(filterQuery); q.setFields(Server.Schema.LANGUAGE.toString()); QueryResponse response = solrServer.query(q, METHOD.POST); diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Ingester.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Ingester.java index be0b93088d..576b65d581 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Ingester.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Ingester.java @@ -97,7 +97,7 @@ class Ingester { * file, but the Solr server is probably fine. */ void indexMetaDataOnly(AbstractFile file) throws IngesterException { - indexChunk("", file.getName().toLowerCase(), new HashMap<>(getContentFields(file))); + indexChunk("", "", file.getName().toLowerCase(), new HashMap<>(getContentFields(file))); } /** @@ -111,7 +111,7 @@ class Ingester { * artifact, but the Solr server is probably fine. */ void indexMetaDataOnly(BlackboardArtifact artifact, String sourceName) throws IngesterException { - indexChunk("", sourceName, new HashMap<>(getContentFields(artifact))); + indexChunk("", "", sourceName, new HashMap<>(getContentFields(artifact))); } /** @@ -156,7 +156,7 @@ class Ingester { logger.log(Level.INFO, "File ingest cancelled. Cancelling keyword search indexing of {0}", sourceName); return false; } - + Chunk chunk = chunker.next(); Map fields = new HashMap<>(contentFields); String chunkId = Server.getChunkIdString(sourceID, numChunks + 1); @@ -166,7 +166,7 @@ class Ingester { language.ifPresent(lang -> languageSpecificContentIndexingHelper.updateLanguageSpecificFields(fields, chunk, lang)); try { //add the chunk text to Solr index - indexChunk(chunk.toString(), sourceName, fields); + indexChunk(chunk.toString(), chunk.geLowerCasedChunk(), sourceName, fields); // add mini chunk when there's a language specific field if (chunker.hasNext() && language.isPresent()) { languageSpecificContentIndexingHelper.indexMiniChunk(chunk, sourceName, new HashMap<>(contentFields), chunkId, language.get()); @@ -197,7 +197,7 @@ class Ingester { fields.put(Server.Schema.ID.toString(), Long.toString(sourceID)); //"parent" docs don't have chunk_size fields.remove(Server.Schema.CHUNK_SIZE.toString()); - indexChunk(null, sourceName, fields); + indexChunk(null, null, sourceName, fields); } } return true; @@ -211,12 +211,13 @@ class Ingester { * 4.0.0), see if possible to stream with UpdateRequestHandler * * @param chunk The chunk content as a string, or null for metadata only + * @param lowerCasedChunk The lower cased chunk content as a string, or null for metadata only * @param fields * @param size * * @throws org.sleuthkit.autopsy.keywordsearch.Ingester.IngesterException */ - private void indexChunk(String chunk, String sourceName, Map fields) throws IngesterException { + private void indexChunk(String chunk, String lowerCasedChunk, String sourceName, Map fields) throws IngesterException { if (fields.get(Server.Schema.IMAGE_ID.toString()) == null) { //JMTODO: actually if the we couldn't get the image id it is set to -1, // but does this really mean we don't want to index it? @@ -245,7 +246,7 @@ class Ingester { // insensitive substring/regular expression search. double indexSchemaVersion = NumberUtils.toDouble(solrServer.getIndexInfo().getSchemaVersion()); if (indexSchemaVersion >= 2.1) { - updateDoc.addField(Server.Schema.CONTENT_STR.toString(), ((chunk == null) ? "" : chunk.toLowerCase())); + updateDoc.addField(Server.Schema.CONTENT_STR.toString(), ((chunk == null) ? "" : lowerCasedChunk)); } TimingMetric metric = HealthMonitor.getTimingMetric("Solr: Index chunk"); diff --git a/README.txt b/README.txt index 73abd3e64e..3ed2dda963 100644 --- a/README.txt +++ b/README.txt @@ -72,11 +72,11 @@ GStreamer for viewing video files - Web page: http://gstreamer.freedesktop.org/ - License: http://www.gnu.org/licenses/lgpl.html -GStreamer-java for viewing video files -- Web page: http://code.google.com/p/gstreamer-java/ -- License: http://www.gnu.org/licenses/lgpl.html +GStreamer 1.x Java Core for viewing video files +- Web page: https://github.com/gstreamer-java/gst1-java-core +- License: https://github.com/gstreamer-java/gst1-java-core/blob/master/LICENSE.md -Regripper for pulling recently activity +Regripper for pulling recent activity (Including custom plugins) - Web page: http://regripper.wordpress.com/ - License: http://www.gnu.org/licenses/gpl.html diff --git a/Running_Linux_OSX.txt b/Running_Linux_OSX.txt index 7244c6320d..56ea6a5888 100644 --- a/Running_Linux_OSX.txt +++ b/Running_Linux_OSX.txt @@ -12,19 +12,23 @@ The following need to be done at least once. They do not need to be repeated for - Install a Java 8 JRE and JavaFX 8 and set JAVA_HOME. -- Linux: Any Java 8 version of OpenJDK/OpenJFX distribution should suffice. The following instructions use the Zulu Community distribution. 1. Download a 64 bit Java 8 JRE for your specific platform from https://www.azul.com/downloads/zulu-community - 2. Install the JRE. e.g. 'sudo apt install ./zulu8.40.0.25-ca-jre8.0.222-linux_amd64.deb' + 2. Install the JRE. e.g. % sudo apt install ./zulu8.40.0.25-ca-jre8.0.222-linux_amd64.deb 3. Download a 64 bit Java 8 JavaFX for your specific platform from the same location. 4. Extract the contents of the JavaFX archive into the folder where the JRE was installed. - e.g. 'cd /usr/lib/jvm/zre-8-amd64; sudo tar xzf ~/Downloads/zulu8.40.0.25-ca-fx-jre8.0.222-linux_x64.tar.gz --strip-components=1' - 5. Confirm Java 8 is being found by running 'java -version' - 6. Set JAVA_HOME environment variable to location of JRE installation (e.g. /usr/lib/jvm/zre-8-amd64) + e.g. % sudo tar xzf ~/Downloads/zulu8.40.0.25-ca-fx-jre8.0.222-linux_x64.tar.gz -C /usr/lib/jvm/zre-8-amd64 --strip-components=1 NOTE: You may need to log out and back in again after setting JAVA_HOME before the Autopsy unix_setup.sh script can see the value. --- OS X: Use The Oracle website: https://www.java.com/ - Set JAVA_HOME with something like: export JAVA_HOME=`/usr/libexec/java_home` in .bash_profile - +-- OS X: Any Java 8 version of OpenJDK/OpenJFX distribution should suffice. + 1. Install a 64 bit Java 8 JRE. + % brew tap adoptopenjdk/openjdk + % brew cask install adoptopenjdk8 + 2. Download a 64 bit Java 8 JavaFX for macOS from https://www.azul.com/downloads/zulu-community + 3. Extract the contents of the JavaFX archive into the folder where the JRE was installed. + e.g. % sudo tar xf ~/Downloads/zulu8.40.0.25-ca-fx-jre8.0.222-macosx_x64.tar.gz -C /Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home--strip-components=1 + 4. Confirm Java 8 is being found by running 'java -version' + 5. Set JAVA_HOME environment variable to location of JRE installation. * Install The Sleuth Kit Java Bindings * diff --git a/docs/doxygen/debugTsk.dox b/docs/doxygen/debugTsk.dox index d70c014667..5e8ab961b2 100644 --- a/docs/doxygen/debugTsk.dox +++ b/docs/doxygen/debugTsk.dox @@ -3,7 +3,7 @@ If you find that you need to debug some of the C/C++ code from The Sleuth Kit (TSK), then here are the steps to follow: -# Ensure that you have the Debug version of the TSK JNI dll built (both 32-bit and 64-bit to be safe). This assumes you built TSK from source and are not simply using the developer platform. You may have to build the libtsk_jni twice because sommetimes it complains about not being able to find a .map file. --# Run the 'dist-debug' target for the TSK DataModel project. This copies the debug versions of the dll into the JAR file. If you run the 'dist' target, then you will get Release versions of the dll and you won't have the needed symbols for debugging. +-# Run the 'Debug-PostgreSQL' target for the TSK DataModel project. This copies the debug versions of the dll into the JAR file. If you run the 'dist' target, then you will get Release versions of the dll and you won't have the needed symbols for debugging. -# Build the Autopsy suite so that it copies the new JAR file with the debug dlls. -# Set your breakpoints in the TSK source. -# Run Autopsy in the debugger. diff --git a/pythonExamples/Aug2015DataSourceTutorial/FindContactsDb.py b/pythonExamples/Aug2015DataSourceTutorial/FindContactsDb.py index 69a103dcdf..e03ad34121 100644 --- a/pythonExamples/Aug2015DataSourceTutorial/FindContactsDb.py +++ b/pythonExamples/Aug2015DataSourceTutorial/FindContactsDb.py @@ -98,15 +98,15 @@ class ContactsDbIngestModule(DataSourceIngestModule): # Where any setup and configuration is done # 'context' is an instance of org.sleuthkit.autopsy.ingest.IngestJobContext. - # See: http://sleuthkit.org/autopsy/docs/api-docs/4.6.0/classorg_1_1sleuthkit_1_1autopsy_1_1ingest_1_1_ingest_job_context.html + # See: http://sleuthkit.org/autopsy/docs/api-docs/4.13.0/classorg_1_1sleuthkit_1_1autopsy_1_1ingest_1_1_ingest_job_context.html def startUp(self, context): self.context = context # Where the analysis is done. # The 'dataSource' object being passed in is of type org.sleuthkit.datamodel.Content. - # See: http://www.sleuthkit.org/sleuthkit/docs/jni-docs/4.6.0/interfaceorg_1_1sleuthkit_1_1datamodel_1_1_content.html + # See: http://www.sleuthkit.org/sleuthkit/docs/jni-docs/4.13.0/interfaceorg_1_1sleuthkit_1_1datamodel_1_1_content.html # 'progressBar' is of type org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProgress - # See: http://sleuthkit.org/autopsy/docs/api-docs/4.6.0/classorg_1_1sleuthkit_1_1autopsy_1_1ingest_1_1_data_source_ingest_module_progress.html + # See: http://sleuthkit.org/autopsy/docs/api-docs/4.13.0/classorg_1_1sleuthkit_1_1autopsy_1_1ingest_1_1_data_source_ingest_module_progress.html def process(self, dataSource, progressBar): # we don't know how much work there is yet diff --git a/pythonExamples/July2015FileTutorial_BigRound/FindBigRoundFiles.py b/pythonExamples/July2015FileTutorial_BigRound/FindBigRoundFiles.py index 712cbd24c8..b2b13db96f 100644 --- a/pythonExamples/July2015FileTutorial_BigRound/FindBigRoundFiles.py +++ b/pythonExamples/July2015FileTutorial_BigRound/FindBigRoundFiles.py @@ -92,7 +92,7 @@ class FindBigRoundFilesIngestModule(FileIngestModule): # Where any setup and configuration is done # 'context' is an instance of org.sleuthkit.autopsy.ingest.IngestJobContext. - # See: http://sleuthkit.org/autopsy/docs/api-docs/4.6.0/classorg_1_1sleuthkit_1_1autopsy_1_1ingest_1_1_ingest_job_context.html + # See: http://sleuthkit.org/autopsy/docs/api-docs/4.13.0/classorg_1_1sleuthkit_1_1autopsy_1_1ingest_1_1_ingest_job_context.html # TODO: Add any setup code that you need here. def startUp(self, context): self.filesFound = 0 @@ -103,7 +103,7 @@ class FindBigRoundFilesIngestModule(FileIngestModule): # Where the analysis is done. Each file will be passed into here. # The 'file' object being passed in is of type org.sleuthkit.datamodel.AbstractFile. - # See: http://www.sleuthkit.org/sleuthkit/docs/jni-docs/4.6.0/classorg_1_1sleuthkit_1_1datamodel_1_1_abstract_file.html + # See: http://www.sleuthkit.org/sleuthkit/docs/jni-docs/4.13.0/classorg_1_1sleuthkit_1_1datamodel_1_1_abstract_file.html def process(self, file): # Use blackboard class to index blackboard artifacts for keyword search diff --git a/pythonExamples/Sept2015ReportTutorial_CSV/CsvReportModule.py b/pythonExamples/Sept2015ReportTutorial_CSV/CsvReportModule.py index 5e21c125bd..9253e0ee82 100644 --- a/pythonExamples/Sept2015ReportTutorial_CSV/CsvReportModule.py +++ b/pythonExamples/Sept2015ReportTutorial_CSV/CsvReportModule.py @@ -27,7 +27,7 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. -# See http://sleuthkit.org/autopsy/docs/api-docs/4.6.0/index.html for documentation +# See http://sleuthkit.org/autopsy/docs/api-docs/4.13.0/index.html for documentation # Simple report module for Autopsy. # Used as part of Python tutorials from Basis Technology - September 2015 @@ -71,7 +71,7 @@ class CSVReportModule(GeneralReportModuleAdapter): # TODO: Update this method to make a report # The 'baseReportDir' object being passed in is a string with the directory that reports are being stored in. Report should go into baseReportDir + getRelativeFilePath(). # The 'progressBar' object is of type ReportProgressPanel. - # See: http://sleuthkit.org/autopsy/docs/api-docs/4.6.0/classorg_1_1sleuthkit_1_1autopsy_1_1report_1_1_report_progress_panel.html + # See: http://sleuthkit.org/autopsy/docs/api-docs/4.13.0/classorg_1_1sleuthkit_1_1autopsy_1_1report_1_1_report_progress_panel.html def generateReport(self, baseReportDir, progressBar): # Open the output file. diff --git a/pythonExamples/dataSourceIngestModule.py b/pythonExamples/dataSourceIngestModule.py index c2edfcd8fb..07e5520e74 100644 --- a/pythonExamples/dataSourceIngestModule.py +++ b/pythonExamples/dataSourceIngestModule.py @@ -29,7 +29,7 @@ # Simple data source-level ingest module for Autopsy. # Search for TODO for the things that you need to change -# See http://sleuthkit.org/autopsy/docs/api-docs/4.6.0/index.html for documentation +# See http://sleuthkit.org/autopsy/docs/api-docs/4.13.0/index.html for documentation import jarray import inspect @@ -94,7 +94,7 @@ class SampleJythonDataSourceIngestModule(DataSourceIngestModule): # Where any setup and configuration is done # 'context' is an instance of org.sleuthkit.autopsy.ingest.IngestJobContext. - # See: http://sleuthkit.org/autopsy/docs/api-docs/4.6.0/classorg_1_1sleuthkit_1_1autopsy_1_1ingest_1_1_ingest_job_context.html + # See: http://sleuthkit.org/autopsy/docs/api-docs/4.13.0/classorg_1_1sleuthkit_1_1autopsy_1_1ingest_1_1_ingest_job_context.html # TODO: Add any setup code that you need here. def startUp(self, context): @@ -104,9 +104,9 @@ class SampleJythonDataSourceIngestModule(DataSourceIngestModule): # Where the analysis is done. # The 'dataSource' object being passed in is of type org.sleuthkit.datamodel.Content. - # See: http://www.sleuthkit.org/sleuthkit/docs/jni-docs/4.6.0/interfaceorg_1_1sleuthkit_1_1datamodel_1_1_content.html + # See: http://www.sleuthkit.org/sleuthkit/docs/jni-docs/4.13.0/interfaceorg_1_1sleuthkit_1_1datamodel_1_1_content.html # 'progressBar' is of type org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProgress - # See: http://sleuthkit.org/autopsy/docs/api-docs/4.6.0/classorg_1_1sleuthkit_1_1autopsy_1_1ingest_1_1_data_source_ingest_module_progress.html + # See: http://sleuthkit.org/autopsy/docs/api-docs/4.13.0/classorg_1_1sleuthkit_1_1autopsy_1_1ingest_1_1_data_source_ingest_module_progress.html # TODO: Add your analysis code in here. def process(self, dataSource, progressBar): @@ -119,7 +119,7 @@ class SampleJythonDataSourceIngestModule(DataSourceIngestModule): # For our example, we will use FileManager to get all # files with the word "test" # in the name and then count and read them - # FileManager API: http://sleuthkit.org/autopsy/docs/api-docs/4.6.0/classorg_1_1sleuthkit_1_1autopsy_1_1casemodule_1_1services_1_1_file_manager.html + # FileManager API: http://sleuthkit.org/autopsy/docs/api-docs/4.13.0/classorg_1_1sleuthkit_1_1autopsy_1_1casemodule_1_1services_1_1_file_manager.html fileManager = Case.getCurrentCase().getServices().getFileManager() files = fileManager.findFiles(dataSource, "%test%") diff --git a/pythonExamples/fileIngestModule.py b/pythonExamples/fileIngestModule.py index b36eea7c57..078623c61d 100644 --- a/pythonExamples/fileIngestModule.py +++ b/pythonExamples/fileIngestModule.py @@ -29,7 +29,7 @@ # Simple file-level ingest module for Autopsy. # Search for TODO for the things that you need to change -# See http://sleuthkit.org/autopsy/docs/api-docs/4.6.0/index.html for documentation +# See http://sleuthkit.org/autopsy/docs/api-docs/4.13.0/index.html for documentation import jarray import inspect @@ -94,7 +94,7 @@ class SampleJythonFileIngestModule(FileIngestModule): # Where any setup and configuration is done # 'context' is an instance of org.sleuthkit.autopsy.ingest.IngestJobContext. - # See: http://sleuthkit.org/autopsy/docs/api-docs/4.6.0/classorg_1_1sleuthkit_1_1autopsy_1_1ingest_1_1_ingest_job_context.html + # See: http://sleuthkit.org/autopsy/docs/api-docs/4.13.0/classorg_1_1sleuthkit_1_1autopsy_1_1ingest_1_1_ingest_job_context.html # TODO: Add any setup code that you need here. def startUp(self, context): self.filesFound = 0 @@ -105,7 +105,7 @@ class SampleJythonFileIngestModule(FileIngestModule): # Where the analysis is done. Each file will be passed into here. # The 'file' object being passed in is of type org.sleuthkit.datamodel.AbstractFile. - # See: http://www.sleuthkit.org/sleuthkit/docs/jni-docs/4.6.0/classorg_1_1sleuthkit_1_1datamodel_1_1_abstract_file.html + # See: http://www.sleuthkit.org/sleuthkit/docs/jni-docs/4.13.0/classorg_1_1sleuthkit_1_1datamodel_1_1_abstract_file.html # TODO: Add your analysis code in here. def process(self, file): # Skip non-files diff --git a/pythonExamples/fileIngestModuleWithGui.py b/pythonExamples/fileIngestModuleWithGui.py index cdcaca3576..a7daf5d982 100644 --- a/pythonExamples/fileIngestModuleWithGui.py +++ b/pythonExamples/fileIngestModuleWithGui.py @@ -35,7 +35,7 @@ # don't need a configuration UI, start with the other sample module. # # Search for TODO for the things that you need to change -# See http://sleuthkit.org/autopsy/docs/api-docs/4.6.0/index.html for documentation +# See http://sleuthkit.org/autopsy/docs/api-docs/4.13.0/index.html for documentation import jarray diff --git a/pythonExamples/reportmodule.py b/pythonExamples/reportmodule.py index 1011cfe753..d1f0e927a6 100644 --- a/pythonExamples/reportmodule.py +++ b/pythonExamples/reportmodule.py @@ -31,7 +31,7 @@ # Sample report module for Autopsy. Use as a starting point for new modules. # # Search for TODO for the things that you need to change -# See http://sleuthkit.org/autopsy/docs/api-docs/4.6.0/index.html for documentation +# See http://sleuthkit.org/autopsy/docs/api-docs/4.13.0/index.html for documentation import os from java.lang import System @@ -69,7 +69,7 @@ class SampleGeneralReportModule(GeneralReportModuleAdapter): # TODO: Update this method to make a report # The 'baseReportDir' object being passed in is a string with the directory that reports are being stored in. Report should go into baseReportDir + getRelativeFilePath(). # The 'progressBar' object is of type ReportProgressPanel. - # See: http://sleuthkit.org/autopsy/docs/api-docs/4.6.0/classorg_1_1sleuthkit_1_1autopsy_1_1report_1_1_report_progress_panel.html + # See: http://sleuthkit.org/autopsy/docs/api-docs/4.13.0/classorg_1_1sleuthkit_1_1autopsy_1_1report_1_1_report_progress_panel.html def generateReport(self, baseReportDir, progressBar): # For an example, we write a file with the number of files created in the past 2 weeks