2021-07-29 11:46:58 +00:00
|
|
|
# Mobile Verification Toolkit (MVT)
|
2021-08-01 19:11:08 +00:00
|
|
|
# Copyright (c) 2021 The MVT Project Authors.
|
|
|
|
# Use of this software is governed by the MVT License 1.1 that can be found at
|
|
|
|
# https://license.mvt.re/1.1/
|
2021-07-29 11:46:58 +00:00
|
|
|
|
2021-07-30 09:40:09 +00:00
|
|
|
import datetime
|
2021-07-29 11:46:58 +00:00
|
|
|
import os
|
|
|
|
import sqlite3
|
|
|
|
|
2021-08-14 16:56:33 +00:00
|
|
|
from mvt.common.utils import convert_timestamp_to_iso
|
2021-07-29 11:46:58 +00:00
|
|
|
|
2021-08-15 11:14:18 +00:00
|
|
|
from ..base import IOSExtraction
|
2021-07-30 09:40:09 +00:00
|
|
|
|
2021-07-29 11:46:58 +00:00
|
|
|
WEBKIT_RESOURCELOADSTATICS_BACKUP_RELPATH = "Library/WebKit/WebsiteData/ResourceLoadStatistics/observations.db"
|
|
|
|
WEBKIT_RESOURCELOADSTATICS_ROOT_PATHS = [
|
|
|
|
"private/var/mobile/Containers/Data/Application/*/Library/WebKit/WebsiteData/ResourceLoadStatistics/observations.db",
|
|
|
|
"private/var/mobile/Containers/Data/Application/*/SystemData/com.apple.SafariViewService/Library/WebKit/WebsiteData/observations.db",
|
|
|
|
]
|
|
|
|
|
2021-11-19 14:27:51 +00:00
|
|
|
|
2021-07-29 11:46:58 +00:00
|
|
|
class WebkitResourceLoadStatistics(IOSExtraction):
|
2021-09-10 13:18:13 +00:00
|
|
|
"""This module extracts records from WebKit ResourceLoadStatistics observations.db."""
|
2021-07-29 11:46:58 +00:00
|
|
|
# TODO: Add serialize().
|
|
|
|
|
|
|
|
def __init__(self, file_path=None, base_folder=None, output_folder=None,
|
|
|
|
fast_mode=False, log=None, results=[]):
|
|
|
|
super().__init__(file_path=file_path, base_folder=base_folder,
|
|
|
|
output_folder=output_folder, fast_mode=fast_mode,
|
|
|
|
log=log, results=results)
|
|
|
|
|
2021-08-15 16:53:02 +00:00
|
|
|
self.results = {}
|
|
|
|
|
2021-07-29 11:46:58 +00:00
|
|
|
def check_indicators(self):
|
|
|
|
if not self.indicators:
|
|
|
|
return
|
|
|
|
|
|
|
|
self.detected = {}
|
|
|
|
for key, items in self.results.items():
|
|
|
|
for item in items:
|
|
|
|
if self.indicators.check_domain(item["registrable_domain"]):
|
|
|
|
if key not in self.detected:
|
2021-11-19 14:27:51 +00:00
|
|
|
self.detected[key] = [item, ]
|
2021-07-29 11:46:58 +00:00
|
|
|
else:
|
|
|
|
self.detected[key].append(item)
|
|
|
|
|
|
|
|
def _process_observations_db(self, db_path, key):
|
|
|
|
self.log.info("Found WebKit ResourceLoadStatistics observations.db file at path %s", db_path)
|
|
|
|
|
2021-08-16 08:50:35 +00:00
|
|
|
self._recover_sqlite_db_if_needed(db_path)
|
2021-07-29 11:46:58 +00:00
|
|
|
|
|
|
|
conn = sqlite3.connect(db_path)
|
|
|
|
cur = conn.cursor()
|
|
|
|
|
|
|
|
try:
|
|
|
|
cur.execute("SELECT * from ObservedDomains;")
|
|
|
|
except sqlite3.OperationalError:
|
|
|
|
return
|
|
|
|
|
2021-11-19 14:27:51 +00:00
|
|
|
if key not in self.results:
|
2021-07-29 11:46:58 +00:00
|
|
|
self.results[key] = []
|
|
|
|
|
|
|
|
for row in cur:
|
2021-08-15 17:20:41 +00:00
|
|
|
self.results[key].append({
|
|
|
|
"domain_id": row[0],
|
|
|
|
"registrable_domain": row[1],
|
|
|
|
"last_seen": row[2],
|
|
|
|
"had_user_interaction": bool(row[3]),
|
|
|
|
"last_seen_isodate": convert_timestamp_to_iso(datetime.datetime.utcfromtimestamp(int(row[2]))),
|
|
|
|
})
|
2021-07-29 11:46:58 +00:00
|
|
|
|
|
|
|
if len(self.results[key]) > 0:
|
|
|
|
self.log.info("Extracted a total of %d records from %s", len(self.results[key]), db_path)
|
|
|
|
|
|
|
|
def run(self):
|
|
|
|
if self.is_backup:
|
|
|
|
try:
|
2021-08-15 18:00:29 +00:00
|
|
|
for backup_file in self._get_backup_files_from_manifest(relative_path=WEBKIT_RESOURCELOADSTATICS_BACKUP_RELPATH):
|
2021-09-01 19:59:12 +00:00
|
|
|
db_path = self._get_backup_file_from_id(backup_file["file_id"])
|
2021-08-15 15:39:14 +00:00
|
|
|
key = f"{backup_file['domain']}/{WEBKIT_RESOURCELOADSTATICS_BACKUP_RELPATH}"
|
|
|
|
self._process_observations_db(db_path=db_path, key=key)
|
2021-07-29 11:46:58 +00:00
|
|
|
except Exception as e:
|
2021-08-15 15:39:14 +00:00
|
|
|
self.log.info("Unable to search for WebKit observations.db: %s", e)
|
2021-07-29 11:46:58 +00:00
|
|
|
elif self.is_fs_dump:
|
2021-08-16 08:50:35 +00:00
|
|
|
for db_path in self._get_fs_files_from_patterns(WEBKIT_RESOURCELOADSTATICS_ROOT_PATHS):
|
2021-07-29 11:46:58 +00:00
|
|
|
self._process_observations_db(db_path=db_path, key=os.path.relpath(db_path, self.base_folder))
|