From c54ab2c784d7a5941ba5040a2ac0a2b9a2bbb2cb Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Thu, 21 Jan 2021 19:52:23 -0500 Subject: [PATCH 1/7] geolocation start --- .../datamodel/GeolocationSummary.java | 111 +++++++++++++----- 1 file changed, 80 insertions(+), 31 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/GeolocationSummary.java b/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/GeolocationSummary.java index 046c443e4e..3f3b422ac3 100644 --- a/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/GeolocationSummary.java +++ b/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/GeolocationSummary.java @@ -268,20 +268,47 @@ public class GeolocationSummary implements DefaultArtifactUpdateGovernor { * @return A pair where the left value is the total count of way points and * the right is the total list of way points that are >= minTime. */ - private Pair getCounts(List points, Long minTime) { - + private Pair getCounts(List points, Long minTime) { if (points == null) { return EMPTY_COUNT; } return points.stream().reduce( EMPTY_COUNT, - (total, w) -> Pair.of(total.getLeft() + 1, total.getRight() + (greaterThanOrEqual(minTime, w.getTimestamp()) ? 1 : 0)), + (total, time) -> Pair.of(total.getLeft() + 1, total.getRight() + (greaterThanOrEqual(minTime, time) ? 1 : 0)), (pair1, pair2) -> Pair.of(pair1.getLeft() + pair2.getLeft(), pair1.getRight() + pair2.getRight())); } private static final long DAY_SECS = 24 * 60 * 60; + + private CityRecord getClosest(ClosestCityMapper cityMapper, MapWaypoint pt) { + if (pt == null) { + return null; + } + + return cityMapper.findClosest(new CityRecord(null, null, null, pt.getX(), pt.getY())); + } + + private Pair getClosestWithTime(ClosestCityMapper cityMapper, MapWaypoint pt) { + if (pt == null) { + return null; + } + + CityRecord city = getClosest(cityMapper, pt); + + Long time = pt.getTimestamp(); + return Pair.of(city, time); + } + + private + + private List> processGeoResult(GeoResult geoResult) throws IOException { + ClosestCityMapper closestCityMapper = ClosestCityMapper.getInstance(); + + Stream> + + } /** * Get this list of hits per city where the list is sorted descending by @@ -300,31 +327,34 @@ public class GeolocationSummary implements DefaultArtifactUpdateGovernor { public CityData getCityCounts(DataSource dataSource, int daysCount, int maxCount) throws SleuthkitCaseProviderException, GeoLocationDataException, InterruptedException, IOException { - ClosestCityMapper closestCityMapper = ClosestCityMapper.getInstance(); - - List dataSourcePoints = getPoints(dataSource); - - Map> allCityPoints = new HashMap<>(); - List others = new ArrayList<>(); + GeoResult geoResult = getGeoResult(dataSource); + List> dataSourcePoints = processGeoResult(geoResult); + + Map> allCityPoints = new HashMap<>(); + List others = new ArrayList<>(); Long mostRecent = null; - for (MapWaypoint pt : dataSourcePoints) { - CityRecord city = closestCityMapper.findClosest(new CityRecord(null, null, null, pt.getX(), pt.getY())); - Long curTime = pt.getTimestamp(); + for (Pair pt : dataSourcePoints) { + if (pt == null) { + continue; + } + + Long curTime = pt.getRight(); if (curTime != null && (mostRecent == null || curTime > mostRecent)) { mostRecent = curTime; } + CityRecord city = pt.getLeft(); if (city == null) { - others.add(pt); + others.add(curTime); } else { - List cityPoints = allCityPoints.get(city); + List cityPoints = allCityPoints.get(city); if (cityPoints == null) { cityPoints = new ArrayList<>(); allCityPoints.put(city, cityPoints); } - cityPoints.add(pt); + cityPoints.add(curTime); } } @@ -355,14 +385,41 @@ public class GeolocationSummary implements DefaultArtifactUpdateGovernor { new CityCountsList(mostRecentCounts, otherMostRecentCount), mostRecentMinTime); } + + + private static class GeoResult { + private final Set mapWaypoints; + private final List> tracks; + private final List> areas; + + private GeoResult(Set mapWaypoints, List> tracks, List> areas) { + this.mapWaypoints = mapWaypoints; + this.tracks = tracks; + this.areas = areas; + } + + private Set getMapWaypoints() { + return mapWaypoints; + } + + private List> getTracks() { + return tracks; + } + + private List> getAreas() { + return areas; + } + + + } /** * Means of fetching points from geolocation. */ private static class PointFetcher extends AbstractWaypointFetcher { - private final BlockingQueue> asyncResult; - + private final BlockingQueue asyncResult; + /** * Main constructor. * @@ -371,26 +428,16 @@ public class GeolocationSummary implements DefaultArtifactUpdateGovernor { * BlockingQueue blocks until a result is received from geolocation. * @param filters The applicable filters for geolocation. */ - public PointFetcher(BlockingQueue> asyncResult, GeoFilter filters) { + public PointFetcher(BlockingQueue asyncResult, GeoFilter filters) { super(filters); this.asyncResult = asyncResult; } @Override public void handleFilteredWaypointSet(Set mapWaypoints, List> tracks, List> areas, boolean wasEntirelySuccessful) { - Stream>> stream = Stream.of( - Arrays.asList(mapWaypoints), - tracks == null ? Collections.emptyList() : tracks, - areas == null ? Collections.emptyList() : areas); - - List wayPoints = stream - .flatMap((List> list) -> list.stream()) - .flatMap((Set set) -> set.stream()) - .collect(Collectors.toList()); - // push to blocking queue to continue try { - asyncResult.put(wayPoints); + asyncResult.put(new GeoResult(mapWaypoints, tracks, areas)); } catch (InterruptedException ignored) { // ignored cancellations } @@ -406,10 +453,12 @@ public class GeolocationSummary implements DefaultArtifactUpdateGovernor { * @throws GeoLocationDataException * @throws InterruptedException */ - private List getPoints(DataSource dataSource) throws SleuthkitCaseProviderException, GeoLocationDataException, InterruptedException { + private GeoResult getGeoResult(DataSource dataSource) + throws SleuthkitCaseProviderException, GeoLocationDataException, InterruptedException { + // make asynchronous callback synchronous (the callback nature will be handled in a different level) // see the following: https://stackoverflow.com/questions/20659961/java-synchronous-callback - final BlockingQueue> asyncResult = new ArrayBlockingQueue<>(1); + final BlockingQueue asyncResult = new ArrayBlockingQueue<>(1); GeoFilter geoFilter = new GeoFilter(true, false, 0, Arrays.asList(dataSource), GPS_ARTIFACT_TYPES); From 03d2a18788c7b5322035748e986b839512362254 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Thu, 21 Jan 2021 20:29:06 -0500 Subject: [PATCH 2/7] updated geolocation --- .../datamodel/GeolocationSummary.java | 37 ++++++++++++++++++- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/GeolocationSummary.java b/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/GeolocationSummary.java index 3f3b422ac3..865fd6a9d0 100644 --- a/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/GeolocationSummary.java +++ b/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/GeolocationSummary.java @@ -23,6 +23,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -301,13 +302,45 @@ public class GeolocationSummary implements DefaultArtifactUpdateGovernor { return Pair.of(city, time); } - private + private Stream> reduceGrouping(Set points, ClosestCityMapper cityMapper) { + if (points == null) { + return Stream.empty(); + } + + Map timeMapping = points.stream() + .map((pt) -> getClosestWithTime(cityMapper, pt)) + .filter((pr) -> pr != null) + .collect(Collectors.toMap( + pair -> pair.getLeft(), + pair -> pair.getRight(), + (time1, time2) -> + // get latest time + ((time1 != null && time2 == null) || (time1 != null && time2 != null && time1 > time2)) ? + time1 : + time2)); + + return timeMapping.entrySet().stream() + .map(e -> Pair.of(e.getKey(), e.getValue())); + + + } private List> processGeoResult(GeoResult geoResult) throws IOException { + if (geoResult == null) { + return Collections.emptyList(); + } + ClosestCityMapper closestCityMapper = ClosestCityMapper.getInstance(); - Stream> + Stream> reducedGroupings = Stream.of(geoResult.getAreas(), geoResult.getTracks()) + .flatMap((groupingList) -> groupingList.stream()) + .flatMap((grouping) -> reduceGrouping(grouping, closestCityMapper)); + Set pointSet = geoResult.getMapWaypoints() == null ? Collections.emptySet() : geoResult.getMapWaypoints(); + Stream> citiesForPoints = pointSet.stream().map(pt -> getClosestWithTime(closestCityMapper, pt)); + + return Stream.concat(reducedGroupings, citiesForPoints) + .collect(Collectors.toList()); } /** From 9de1115cc43b68a8eb12c4e555f4d4bd04e61a86 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Fri, 22 Jan 2021 08:21:44 -0500 Subject: [PATCH 3/7] commenting and formatting --- .../datamodel/GeolocationSummary.java | 206 ++++++++++-------- 1 file changed, 120 insertions(+), 86 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/GeolocationSummary.java b/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/GeolocationSummary.java index 865fd6a9d0..a981d40685 100644 --- a/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/GeolocationSummary.java +++ b/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/GeolocationSummary.java @@ -23,7 +23,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -32,7 +31,6 @@ import java.util.concurrent.BlockingQueue; import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.commons.lang3.tuple.Pair; -import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.datasourcesummary.datamodel.SleuthkitCaseProvider.SleuthkitCaseProviderException; import org.sleuthkit.autopsy.datasourcesummary.uiutils.DefaultArtifactUpdateGovernor; import org.sleuthkit.autopsy.geolocation.AbstractWaypointFetcher; @@ -48,6 +46,7 @@ import org.sleuthkit.datamodel.DataSource; * Gathers summary data about Geolocation information for a data source. */ public class GeolocationSummary implements DefaultArtifactUpdateGovernor { + /** * A count of hits for a particular city. */ @@ -170,6 +169,53 @@ public class GeolocationSummary implements DefaultArtifactUpdateGovernor { } } + /** + * Carries data retrieved from the Geolocation API to be processed for + * closest cities. + */ + private static class GeoResult { + + private final Set mapWaypoints; + private final List> tracks; + private final List> areas; + + /** + * Main constructor. + * + * @param mapWaypoints The way points found for the data source. + * @param tracks A list of sets where each set is a track in the data + * source. + * @param areas A list of areas where each set is an area in the data + * source. + */ + private GeoResult(Set mapWaypoints, List> tracks, List> areas) { + this.mapWaypoints = mapWaypoints; + this.tracks = tracks; + this.areas = areas; + } + + /** + * @return The way points found for the data source. + */ + private Set getMapWaypoints() { + return mapWaypoints; + } + + /** + * @return A list of sets where each set is a track in the data source. + */ + private List> getTracks() { + return tracks; + } + + /** + * @return A list of areas where each set is an area in the data source. + */ + private List> getAreas() { + return areas; + } + } + // taken from GeoFilterPanel: all of the GPS artifact types. @SuppressWarnings("deprecation") private static final List GPS_ARTIFACT_TYPES = Arrays.asList( @@ -187,8 +233,11 @@ public class GeolocationSummary implements DefaultArtifactUpdateGovernor { .map(artifactType -> artifactType.getTypeID()) .collect(Collectors.toSet()); + private static final Pair EMPTY_COUNT = Pair.of(0, 0); + + private static final long DAY_SECS = 24 * 60 * 60; + private final SleuthkitCaseProvider provider; - private final java.util.logging.Logger logger; private final SupplierWithException cityMapper; /** @@ -209,7 +258,7 @@ public class GeolocationSummary implements DefaultArtifactUpdateGovernor { * Default constructor. */ public GeolocationSummary() { - this(() -> ClosestCityMapper.getInstance(), SleuthkitCaseProvider.DEFAULT, Logger.getLogger(GeolocationSummary.class.getName())); + this(() -> ClosestCityMapper.getInstance(), SleuthkitCaseProvider.DEFAULT); } /** @@ -218,12 +267,10 @@ public class GeolocationSummary implements DefaultArtifactUpdateGovernor { * @param cityMapper A means of acquiring a ClosestCityMapper that can throw * an IOException. * @param provider A means of acquiring a SleuthkitCaseProvider. - * @param logger The logger. */ - public GeolocationSummary(SupplierWithException cityMapper, SleuthkitCaseProvider provider, java.util.logging.Logger logger) { + public GeolocationSummary(SupplierWithException cityMapper, SleuthkitCaseProvider provider) { this.cityMapper = cityMapper; this.provider = provider; - this.logger = logger; } /** @@ -257,8 +304,6 @@ public class GeolocationSummary implements DefaultArtifactUpdateGovernor { } } - private static final Pair EMPTY_COUNT = Pair.of(0, 0); - /** * Based on a set of waypoints, determines the count of total waypoints and * a total of waypoints whose time stamp is greater than or equal to @@ -279,68 +324,83 @@ public class GeolocationSummary implements DefaultArtifactUpdateGovernor { (total, time) -> Pair.of(total.getLeft() + 1, total.getRight() + (greaterThanOrEqual(minTime, time) ? 1 : 0)), (pair1, pair2) -> Pair.of(pair1.getLeft() + pair2.getLeft(), pair1.getRight() + pair2.getRight())); } - - - private static final long DAY_SECS = 24 * 60 * 60; - - private CityRecord getClosest(ClosestCityMapper cityMapper, MapWaypoint pt) { - if (pt == null) { - return null; - } - - return cityMapper.findClosest(new CityRecord(null, null, null, pt.getX(), pt.getY())); - } - + + /** + * Retrieves a tuple of the closest city (or null if a closest city cannot + * be determined) and the time stamp of the point in seconds from epoch. If + * the point is null, null is returned. + * + * @param cityMapper The means of mapping a point to the closest city. + * @param pt The geolocation point. + * @return A tuple of the closest city and timestamp in seconds from epoch. + */ private Pair getClosestWithTime(ClosestCityMapper cityMapper, MapWaypoint pt) { if (pt == null) { return null; } - - CityRecord city = getClosest(cityMapper, pt); - + + CityRecord city = cityMapper.findClosest(new CityRecord(null, null, null, pt.getX(), pt.getY())); + Long time = pt.getTimestamp(); return Pair.of(city, time); } - + + /** + * Converts a set of waypoints representing a grouping (i.e. track, area) + * into a stream of the unique cities identified in this grouping and the + * latest time stamp for each grouping. + * + * @param points The points in the grouping. + * @param cityMapper The means of mapping a point to the closest city. + * @return A stream of tuples where each tuple will be a unique city (or + * null if a closest is not determined) and the latest timestamp for each. + */ private Stream> reduceGrouping(Set points, ClosestCityMapper cityMapper) { if (points == null) { return Stream.empty(); } - + Map timeMapping = points.stream() .map((pt) -> getClosestWithTime(cityMapper, pt)) .filter((pr) -> pr != null) .collect(Collectors.toMap( - pair -> pair.getLeft(), - pair -> pair.getRight(), - (time1, time2) -> - // get latest time - ((time1 != null && time2 == null) || (time1 != null && time2 != null && time1 > time2)) ? - time1 : - time2)); - + pair -> pair.getLeft(), + pair -> pair.getRight(), + (time1, time2) + -> // get latest time for each unique city + ((time1 != null && time2 == null) || (time1 != null && time2 != null && time1 > time2)) + ? time1 + : time2)); + return timeMapping.entrySet().stream() .map(e -> Pair.of(e.getKey(), e.getValue())); - - + } - - private List> processGeoResult(GeoResult geoResult) throws IOException { + + /** + * Convert a geo result taken from the Geolocation and convert to a stream + * of tuples where each tuple represents a point with the closest city and + * the time stamp in seconds from epoch. + * + * @param geoResult The result from the Geolocation API. + * @param cityMapper The means of mapping a point to the closest city. + * @return A list of tuples where each tuple represents a point to be + * counted with a combination of the closest city and the timestamp. + * @throws IOException + */ + private Stream> processGeoResult(GeoResult geoResult, ClosestCityMapper cityMapper) { if (geoResult == null) { - return Collections.emptyList(); + return Stream.empty(); } - - ClosestCityMapper closestCityMapper = ClosestCityMapper.getInstance(); - + Stream> reducedGroupings = Stream.of(geoResult.getAreas(), geoResult.getTracks()) .flatMap((groupingList) -> groupingList.stream()) - .flatMap((grouping) -> reduceGrouping(grouping, closestCityMapper)); - + .flatMap((grouping) -> reduceGrouping(grouping, cityMapper)); + Set pointSet = geoResult.getMapWaypoints() == null ? Collections.emptySet() : geoResult.getMapWaypoints(); - Stream> citiesForPoints = pointSet.stream().map(pt -> getClosestWithTime(closestCityMapper, pt)); - - return Stream.concat(reducedGroupings, citiesForPoints) - .collect(Collectors.toList()); + Stream> citiesForPoints = pointSet.stream().map(pt -> getClosestWithTime(cityMapper, pt)); + + return Stream.concat(reducedGroupings, citiesForPoints); } /** @@ -348,11 +408,11 @@ public class GeolocationSummary implements DefaultArtifactUpdateGovernor { * number of found hits (i.e. most hits is first index). * * @param dataSource The data source. - * @param daysCount Number of days to go back. - * @param maxCount Maximum number of results. - * + * @param daysCount Number of days to go back. + * @param maxCount Maximum number of results. + * * @return The sorted list. - * + * * @throws SleuthkitCaseProviderException * @throws GeoLocationDataException * @throws InterruptedException @@ -361,8 +421,9 @@ public class GeolocationSummary implements DefaultArtifactUpdateGovernor { throws SleuthkitCaseProviderException, GeoLocationDataException, InterruptedException, IOException { GeoResult geoResult = getGeoResult(dataSource); - List> dataSourcePoints = processGeoResult(geoResult); - + List> dataSourcePoints = processGeoResult(geoResult, this.cityMapper.get()) + .collect(Collectors.toList()); + Map> allCityPoints = new HashMap<>(); List others = new ArrayList<>(); Long mostRecent = null; @@ -371,13 +432,13 @@ public class GeolocationSummary implements DefaultArtifactUpdateGovernor { if (pt == null) { continue; } - + Long curTime = pt.getRight(); if (curTime != null && (mostRecent == null || curTime > mostRecent)) { mostRecent = curTime; } - CityRecord city = pt.getLeft(); + CityRecord city = pt.getLeft(); if (city == null) { others.add(curTime); } else { @@ -418,33 +479,6 @@ public class GeolocationSummary implements DefaultArtifactUpdateGovernor { new CityCountsList(mostRecentCounts, otherMostRecentCount), mostRecentMinTime); } - - - private static class GeoResult { - private final Set mapWaypoints; - private final List> tracks; - private final List> areas; - - private GeoResult(Set mapWaypoints, List> tracks, List> areas) { - this.mapWaypoints = mapWaypoints; - this.tracks = tracks; - this.areas = areas; - } - - private Set getMapWaypoints() { - return mapWaypoints; - } - - private List> getTracks() { - return tracks; - } - - private List> getAreas() { - return areas; - } - - - } /** * Means of fetching points from geolocation. @@ -452,7 +486,7 @@ public class GeolocationSummary implements DefaultArtifactUpdateGovernor { private static class PointFetcher extends AbstractWaypointFetcher { private final BlockingQueue asyncResult; - + /** * Main constructor. * @@ -486,9 +520,9 @@ public class GeolocationSummary implements DefaultArtifactUpdateGovernor { * @throws GeoLocationDataException * @throws InterruptedException */ - private GeoResult getGeoResult(DataSource dataSource) + private GeoResult getGeoResult(DataSource dataSource) throws SleuthkitCaseProviderException, GeoLocationDataException, InterruptedException { - + // make asynchronous callback synchronous (the callback nature will be handled in a different level) // see the following: https://stackoverflow.com/questions/20659961/java-synchronous-callback final BlockingQueue asyncResult = new ArrayBlockingQueue<>(1); From 8e4d7d239774814b6e1bd841a05c18e579d5e214 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Fri, 22 Jan 2021 08:57:07 -0500 Subject: [PATCH 4/7] minor change --- .../datasourcesummary/datamodel/GeolocationSummary.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/GeolocationSummary.java b/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/GeolocationSummary.java index a981d40685..c6c6379083 100644 --- a/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/GeolocationSummary.java +++ b/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/GeolocationSummary.java @@ -420,8 +420,9 @@ public class GeolocationSummary implements DefaultArtifactUpdateGovernor { public CityData getCityCounts(DataSource dataSource, int daysCount, int maxCount) throws SleuthkitCaseProviderException, GeoLocationDataException, InterruptedException, IOException { + ClosestCityMapper closestCityMapper = this.cityMapper.get(); GeoResult geoResult = getGeoResult(dataSource); - List> dataSourcePoints = processGeoResult(geoResult, this.cityMapper.get()) + List> dataSourcePoints = processGeoResult(geoResult, closestCityMapper) .collect(Collectors.toList()); Map> allCityPoints = new HashMap<>(); From cd817b23da9d0aa0a253a3fb905dafe0bd649a11 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Fri, 22 Jan 2021 09:20:44 -0500 Subject: [PATCH 5/7] fix to remove points belonging to tracks or areas --- .../datamodel/GeolocationSummary.java | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/GeolocationSummary.java b/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/GeolocationSummary.java index c6c6379083..ddf1a76533 100644 --- a/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/GeolocationSummary.java +++ b/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/GeolocationSummary.java @@ -225,7 +225,8 @@ public class GeolocationSummary implements DefaultArtifactUpdateGovernor { BlackboardArtifact.ARTIFACT_TYPE.TSK_GPS_SEARCH, BlackboardArtifact.ARTIFACT_TYPE.TSK_GPS_TRACK, BlackboardArtifact.ARTIFACT_TYPE.TSK_GPS_TRACKPOINT, - BlackboardArtifact.ARTIFACT_TYPE.TSK_METADATA_EXIF + BlackboardArtifact.ARTIFACT_TYPE.TSK_METADATA_EXIF, + BlackboardArtifact.ARTIFACT_TYPE.TSK_GPS_AREA ); // all GPS types @@ -393,12 +394,24 @@ public class GeolocationSummary implements DefaultArtifactUpdateGovernor { return Stream.empty(); } - Stream> reducedGroupings = Stream.of(geoResult.getAreas(), geoResult.getTracks()) + List> areas = (geoResult.getAreas() == null) ? Collections.emptyList() : geoResult.getAreas(); + List> tracks = (geoResult.getTracks() == null) ? Collections.emptyList() : geoResult.getTracks(); + + Stream> reducedGroupings = Stream.of(areas, tracks) .flatMap((groupingList) -> groupingList.stream()) .flatMap((grouping) -> reduceGrouping(grouping, cityMapper)); + final Set allTracksAndAreas = Stream.of(areas, tracks) + .flatMap((groupingList) -> groupingList.stream()) + .flatMap((group) -> group.stream()) + .collect(Collectors.toSet()); + Set pointSet = geoResult.getMapWaypoints() == null ? Collections.emptySet() : geoResult.getMapWaypoints(); - Stream> citiesForPoints = pointSet.stream().map(pt -> getClosestWithTime(cityMapper, pt)); + Stream> citiesForPoints = pointSet.stream() + // it appears that AbstractWaypointFetcher.handleFilteredWaypointSet returns all points + // (including track and area points) in the set of MapWaypoint's. This filters those points out of the remaing. + .filter(pt -> !allTracksAndAreas.contains(pt)) + .map(pt -> getClosestWithTime(cityMapper, pt)); return Stream.concat(reducedGroupings, citiesForPoints); } From 0de2c0092f0a24b10dd8b6e8de2f63489f76c151 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Fri, 22 Jan 2021 09:35:44 -0500 Subject: [PATCH 6/7] comment fix --- .../autopsy/datasourcesummary/datamodel/GeolocationSummary.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/GeolocationSummary.java b/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/GeolocationSummary.java index ddf1a76533..6a14540ac2 100644 --- a/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/GeolocationSummary.java +++ b/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/GeolocationSummary.java @@ -409,7 +409,7 @@ public class GeolocationSummary implements DefaultArtifactUpdateGovernor { Set pointSet = geoResult.getMapWaypoints() == null ? Collections.emptySet() : geoResult.getMapWaypoints(); Stream> citiesForPoints = pointSet.stream() // it appears that AbstractWaypointFetcher.handleFilteredWaypointSet returns all points - // (including track and area points) in the set of MapWaypoint's. This filters those points out of the remaing. + // (including track and area points) in the set of MapWaypoints. This filters those points out of the remaining. .filter(pt -> !allTracksAndAreas.contains(pt)) .map(pt -> getClosestWithTime(cityMapper, pt)); From 3f8e6206f56eb06e69d2640ab7249d1fee8f8cc7 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Fri, 22 Jan 2021 11:26:22 -0500 Subject: [PATCH 7/7] NPE fix --- .../datamodel/GeolocationSummary.java | 28 ++++++++++--------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/GeolocationSummary.java b/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/GeolocationSummary.java index 6a14540ac2..d66bbaf30a 100644 --- a/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/GeolocationSummary.java +++ b/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/GeolocationSummary.java @@ -361,21 +361,23 @@ public class GeolocationSummary implements DefaultArtifactUpdateGovernor { return Stream.empty(); } - Map timeMapping = points.stream() - .map((pt) -> getClosestWithTime(cityMapper, pt)) - .filter((pr) -> pr != null) - .collect(Collectors.toMap( - pair -> pair.getLeft(), - pair -> pair.getRight(), - (time1, time2) - -> // get latest time for each unique city - ((time1 != null && time2 == null) || (time1 != null && time2 != null && time1 > time2)) - ? time1 - : time2)); - + Map timeMapping = new HashMap<>(); + for (MapWaypoint pt : points) { + Pair pair = getClosestWithTime(cityMapper, pt); + if (pair == null) { + continue; + } + + CityRecord city = pair.getLeft(); + Long prevTime = timeMapping.get(city); + Long curTime = pair.getRight(); + if (prevTime == null || (curTime != null && curTime > prevTime)) { + timeMapping.put(city, curTime); + } + } + return timeMapping.entrySet().stream() .map(e -> Pair.of(e.getKey(), e.getValue())); - } /**