Improved analytics iOS versions module, checking dates, and sorting results

mvt/ios/modules/fs/analytics.py

@@ -28,7 +28,7 @@ class Analytics(IOSExtraction):
 
     def serialize(self, record: dict) -> None:
         return {
-            "timestamp": record["timestamp"],
+            "timestamp": record["isodate"],
             "module": self.__class__.__name__,
             "event": record["artifact"],
             "data": f"{record}",
@@ -96,17 +96,17 @@ class Analytics(IOSExtraction):
 
         for row in cur:
             if row[0] and row[1]:
-                timestamp = convert_timestamp_to_iso(convert_mactime_to_unix(row[0], False))
+                isodate = convert_timestamp_to_iso(convert_mactime_to_unix(row[0], False))
                 data = plistlib.loads(row[1])
-                data["timestamp"] = timestamp
+                data["isodate"] = isodate
             elif row[0]:
-                timestamp = convert_timestamp_to_iso(convert_mactime_to_unix(row[0], False))
+                isodate = convert_timestamp_to_iso(convert_mactime_to_unix(row[0], False))
                 data = {}
-                data["timestamp"] = timestamp
+                data["isodate"] = isodate
             elif row[1]:
-                timestamp = ""
+                isodate = ""
                 data = plistlib.loads(row[1])
-                data["timestamp"] = timestamp
+                data["isodate"] = isodate
 
             data["artifact"] = artifact
 
@@ -127,4 +127,4 @@ class Analytics(IOSExtraction):
         self.log.info("Extracted %d records from analytics databases",
                       len(self.results))
 
-        self.results = sorted(self.results, key=lambda entry: entry["timestamp"])
+        self.results = sorted(self.results, key=lambda entry: entry["isodate"])

mvt/ios/modules/fs/analytics_ios_versions.py

@@ -4,6 +4,7 @@
 #   https://license.mvt.re/1.1/
 
 import logging
+from datetime import datetime
 
 from mvt.ios.versions import find_version_by_build
 
@@ -25,7 +26,7 @@ class AnalyticsIOSVersions(IOSExtraction):
 
     def serialize(self, record: dict) -> None:
         return {
-            "timestamp": record["timestamp"],
+            "timestamp": record["isodate"],
             "module": self.__class__.__name__,
             "event": "analytics_ios_version",
             "data": f"Seen iOS version {record['version']} ({record['build']})",
@@ -35,20 +36,38 @@ class AnalyticsIOSVersions(IOSExtraction):
         anl = Analytics(target_path=self.target_path, log=self.log)
         anl.process_analytics_dbs()
 
-        builds = []
+        dt_format = "%Y-%m-%d %H:%M:%S.%f"
+
+        builds = {}
         for result in anl.results:
             build = result.get("build")
-            if not build or build in builds:
+            if not build:
                 continue
 
+            ts = result.get("isodate", None)
+            if not ts:
+                continue
+
+            if build not in builds.keys():
+                builds[build] = ts
+                continue
+
+            result_dt = datetime.strptime(ts, dt_format)
+            cur_dt = datetime.strptime(builds[build], dt_format)
+
+            if result_dt < cur_dt:
+                builds[build] = ts
+
+        for build, ts in builds.items():
             version = find_version_by_build(build)
 
-            self.log.info("iOS version %s (%s) first appeared on %s",
-                          version, build, result["timestamp"])
             self.results.append({
-                "timestamp": result["timestamp"],
-                "version": version,
+                "isodate": ts,
                 "build": build,
+                "version": version,
             })
 
-            builds.append(build)
+        self.results = sorted(self.results, key=lambda entry: entry["isodate"])
+        for result in self.results:
+            self.log.info("iOS version %s (%s) first appeared on %s",
+                          result["version"], result["build"], result["isodate"])