From db26c90bf797c16fdb1cfbc69825b90c91fa40b1 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Mon, 7 Dec 2020 13:31:07 -0500 Subject: [PATCH] updates to use abstract waypoint fetcher --- .../datamodel/GeolocationSummary.java | 99 ++++++++++++------- .../geolocation/AbstractWaypointFetcher.java | 6 +- .../geolocation/GeolocationTopComponent.java | 2 +- .../autopsy/geolocation/MapWaypoint.java | 9 +- 4 files changed, 77 insertions(+), 39 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/GeolocationSummary.java b/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/GeolocationSummary.java index cbb4851231..4f6813d92b 100644 --- a/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/GeolocationSummary.java +++ b/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/GeolocationSummary.java @@ -29,13 +29,15 @@ import java.util.Set; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; import java.util.stream.Collectors; +import java.util.stream.Stream; import org.apache.commons.lang3.tuple.Pair; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.datasourcesummary.datamodel.SleuthkitCaseProvider.SleuthkitCaseProviderException; import org.sleuthkit.autopsy.datasourcesummary.uiutils.DefaultArtifactUpdateGovernor; +import org.sleuthkit.autopsy.geolocation.AbstractWaypointFetcher; +import org.sleuthkit.autopsy.geolocation.GeoFilter; +import org.sleuthkit.autopsy.geolocation.MapWaypoint; import org.sleuthkit.autopsy.geolocation.datamodel.GeoLocationDataException; -import org.sleuthkit.autopsy.geolocation.datamodel.GeoLocationParseResult; -import org.sleuthkit.autopsy.geolocation.datamodel.Waypoint; import org.sleuthkit.autopsy.geolocation.datamodel.WaypointBuilder; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE; @@ -45,11 +47,10 @@ import org.sleuthkit.datamodel.DataSource; * Gathers summary data about Geolocation information for a data source. */ public class GeolocationSummary implements DefaultArtifactUpdateGovernor { - /** * A count of hits for a particular city. */ - public class CityRecordCount { + public static class CityRecordCount { private final CityRecord cityRecord; private final int count; @@ -267,7 +268,7 @@ public class GeolocationSummary implements DefaultArtifactUpdateGovernor { * @return A pair where the left value is the total count of way points and * the right is the total list of way points that are >= minTime. */ - private Pair getCounts(List points, Long minTime) { + private Pair getCounts(List points, Long minTime) { if (points == null) { return EMPTY_COUNT; @@ -278,6 +279,9 @@ public class GeolocationSummary implements DefaultArtifactUpdateGovernor { (total, w) -> Pair.of(total.getLeft() + 1, total.getRight() + (greaterThanOrEqual(minTime, w.getTimestamp()) ? 1 : 0)), (pair1, pair2) -> Pair.of(pair1.getLeft() + pair2.getLeft(), pair1.getRight() + pair2.getRight())); } + + + private static final long DAY_SECS = 24 * 60 * 60; /** * Get this list of hits per city where the list is sorted descending by @@ -294,14 +298,14 @@ public class GeolocationSummary implements DefaultArtifactUpdateGovernor { ClosestCityMapper closestCityMapper = ClosestCityMapper.getInstance(); - List dataSourcePoints = getPoints(dataSource); + List dataSourcePoints = getPoints(dataSource); - Map> allCityPoints = new HashMap<>(); - List others = new ArrayList<>(); + Map> allCityPoints = new HashMap<>(); + List others = new ArrayList<>(); Long mostRecent = null; - for (Waypoint pt : dataSourcePoints) { - CityRecord city = closestCityMapper.findClosest(new CityRecord(null, null, pt.getLatitude(), pt.getLongitude())); + for (MapWaypoint pt : dataSourcePoints) { + CityRecord city = closestCityMapper.findClosest(new CityRecord(null, null, pt.getX(), pt.getY())); Long curTime = pt.getTimestamp(); if (curTime != null && (mostRecent == null || curTime > mostRecent)) { mostRecent = curTime; @@ -310,7 +314,7 @@ public class GeolocationSummary implements DefaultArtifactUpdateGovernor { if (city == null) { others.add(pt); } else { - List cityPoints = allCityPoints.get(city); + List cityPoints = allCityPoints.get(city); if (cityPoints == null) { cityPoints = new ArrayList<>(); allCityPoints.put(city, cityPoints); @@ -320,11 +324,11 @@ public class GeolocationSummary implements DefaultArtifactUpdateGovernor { } } - final Long mostRecentTime = mostRecent; + final Long mostRecentMinTime = (mostRecent == null) ? null : mostRecent - daysCount * DAY_SECS; // pair left is total count and right is count within range (or mostRecent is null) Map> allCityCounts = allCityPoints.entrySet().stream() - .collect(Collectors.toMap((e) -> e.getKey(), (e) -> getCounts(e.getValue(), mostRecentTime))); + .collect(Collectors.toMap((e) -> e.getKey(), (e) -> getCounts(e.getValue(), mostRecentMinTime))); List mostCommonCounts = allCityCounts.entrySet().stream() .map(e -> new CityRecordCount(e.getKey(), e.getValue().getLeft())) @@ -338,14 +342,55 @@ public class GeolocationSummary implements DefaultArtifactUpdateGovernor { .limit(maxCount) .collect(Collectors.toList()); - Pair otherCounts = getCounts(others, mostRecentTime); + Pair otherCounts = getCounts(others, mostRecentMinTime); int otherMostCommonCount = otherCounts.getLeft(); int otherMostRecentCount = otherCounts.getRight(); return new CityData( new CityCountsList(mostCommonCounts, otherMostCommonCount), new CityCountsList(mostRecentCounts, otherMostRecentCount), - mostRecentTime); + mostRecentMinTime); + } + + /** + * Means of fetching points from geolocation. + */ + private static class PointFetcher extends AbstractWaypointFetcher { + + private final BlockingQueue> asyncResult; + + /** + * Main constructor. + * + * @param asyncResult Geolocation fetches results in a callback which is + * already handled by other mechanisms in data source summary. The + * BlockingQueue blocks until a result is received from geolocation. + * @param filters The applicable filters for geolocation. + */ + public PointFetcher(BlockingQueue> asyncResult, GeoFilter filters) { + super(filters); + this.asyncResult = asyncResult; + } + + @Override + public void handleFilteredWaypointSet(Set mapWaypoints, List> tracks, List> areas, boolean wasEntirelySuccessful) { + Stream>> stream = Stream.of( + Arrays.asList(mapWaypoints), + tracks == null ? Collections.emptyList() : tracks, + areas == null ? Collections.emptyList() : areas); + + List wayPoints = stream + .flatMap((List> list) -> list.stream()) + .flatMap((Set set) -> set.stream()) + .collect(Collectors.toList()); + + // push to blocking queue to continue + try { + asyncResult.put(wayPoints); + } catch (InterruptedException ignored) { + // ignored cancellations + } + } } /** @@ -357,18 +402,12 @@ public class GeolocationSummary implements DefaultArtifactUpdateGovernor { * @throws GeoLocationDataException * @throws InterruptedException */ - private List getPoints(DataSource dataSource) throws SleuthkitCaseProviderException, GeoLocationDataException, InterruptedException { + private List getPoints(DataSource dataSource) throws SleuthkitCaseProviderException, GeoLocationDataException, InterruptedException { // make asynchronous callback synchronous (the callback nature will be handled in a different level) // see the following: https://stackoverflow.com/questions/20659961/java-synchronous-callback - final BlockingQueue> asyncResult = new ArrayBlockingQueue<>(1); + final BlockingQueue> asyncResult = new ArrayBlockingQueue<>(1); - final WaypointBuilder.WaypointFilterQueryCallBack callback = (result) -> { - try { - asyncResult.put(result); - } catch (InterruptedException ignored) { - // ignored cancellations - } - }; + GeoFilter geoFilter = new GeoFilter(true, false, 0, Arrays.asList(dataSource), GPS_ARTIFACT_TYPES); WaypointBuilder.getAllWaypoints(provider.get(), Arrays.asList(dataSource), @@ -376,16 +415,8 @@ public class GeolocationSummary implements DefaultArtifactUpdateGovernor { true, -1, false, - callback); + new PointFetcher(asyncResult, geoFilter)); - GeoLocationParseResult result; - - result = asyncResult.take(); - - if (result.isSuccessfullyParsed()) { - return result.getItems(); - } else { - return Collections.emptyList(); - } + return asyncResult.take(); } } diff --git a/Core/src/org/sleuthkit/autopsy/geolocation/AbstractWaypointFetcher.java b/Core/src/org/sleuthkit/autopsy/geolocation/AbstractWaypointFetcher.java index 60cd362d0d..a7fb2c0b0b 100755 --- a/Core/src/org/sleuthkit/autopsy/geolocation/AbstractWaypointFetcher.java +++ b/Core/src/org/sleuthkit/autopsy/geolocation/AbstractWaypointFetcher.java @@ -35,7 +35,7 @@ import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE; /** * The business logic for filtering waypoints. */ -abstract class AbstractWaypointFetcher implements WaypointBuilder.WaypointFilterQueryCallBack { +public abstract class AbstractWaypointFetcher implements WaypointBuilder.WaypointFilterQueryCallBack { private static final Logger logger = Logger.getLogger(AbstractWaypointFetcher.class.getName()); @@ -46,7 +46,7 @@ abstract class AbstractWaypointFetcher implements WaypointBuilder.WaypointFilter * * @param filters */ - AbstractWaypointFetcher(GeoFilter filters) { + protected AbstractWaypointFetcher(GeoFilter filters) { this.filters = filters; } @@ -79,7 +79,7 @@ abstract class AbstractWaypointFetcher implements WaypointBuilder.WaypointFilter * @param tracks The tracks that were successfully parsed. * @param wasEntirelySuccessful True if no errors occurred while processing. */ - abstract void handleFilteredWaypointSet(Set mapWaypoints, List> tracks, + protected abstract void handleFilteredWaypointSet(Set mapWaypoints, List> tracks, List> areas, boolean wasEntirelySuccessful); @Override diff --git a/Core/src/org/sleuthkit/autopsy/geolocation/GeolocationTopComponent.java b/Core/src/org/sleuthkit/autopsy/geolocation/GeolocationTopComponent.java index cbfc32e49e..0ed06339d6 100755 --- a/Core/src/org/sleuthkit/autopsy/geolocation/GeolocationTopComponent.java +++ b/Core/src/org/sleuthkit/autopsy/geolocation/GeolocationTopComponent.java @@ -525,7 +525,7 @@ public final class GeolocationTopComponent extends TopComponent { } @Override - void handleFilteredWaypointSet(Set mapWaypoints, List> tracks, + protected void handleFilteredWaypointSet(Set mapWaypoints, List> tracks, List> areas, boolean wasEntirelySuccessful) { addWaypointsToMap(mapWaypoints, tracks, areas); diff --git a/Core/src/org/sleuthkit/autopsy/geolocation/MapWaypoint.java b/Core/src/org/sleuthkit/autopsy/geolocation/MapWaypoint.java index 8d51dd6a36..e9e7861d17 100755 --- a/Core/src/org/sleuthkit/autopsy/geolocation/MapWaypoint.java +++ b/Core/src/org/sleuthkit/autopsy/geolocation/MapWaypoint.java @@ -63,7 +63,7 @@ import org.sleuthkit.datamodel.TskCoreException; * */ @SuppressWarnings("deprecation") -final class MapWaypoint extends KdTree.XYZPoint implements org.jxmapviewer.viewer.Waypoint { +public final class MapWaypoint extends KdTree.XYZPoint implements org.jxmapviewer.viewer.Waypoint { private static final Logger logger = Logger.getLogger(MapWaypoint.class.getName()); private final static String HTML_PROP_FORMAT = "%s: %s
"; @@ -366,6 +366,13 @@ final class MapWaypoint extends KdTree.XYZPoint implements org.jxmapviewer.viewe return getColor(dataModelWaypoint.getArtifact().getArtifactTypeID()); } + /** + * @return The timestamp associated with this map way point or null. + */ + public Long getTimestamp() { + return dataModelWaypoint.getTimestamp(); + } + /** * An action class for Extracting artifact files. */