diff --git a/mvt/common/module.py b/mvt/common/module.py index f4dffcd..cf391e3 100644 --- a/mvt/common/module.py +++ b/mvt/common/module.py @@ -123,7 +123,7 @@ class MVTModule(object): else: self.timeline_detected.append(record) - # De-duplicate timeline entries + # De-duplicate timeline entries. self.timeline = self.timeline_deduplicate(self.timeline) self.timeline_detected = self.timeline_deduplicate(self.timeline_detected) diff --git a/mvt/ios/modules/fs/__init__.py b/mvt/ios/modules/fs/__init__.py index dad5124..1c72e74 100644 --- a/mvt/ios/modules/fs/__init__.py +++ b/mvt/ios/modules/fs/__init__.py @@ -14,6 +14,7 @@ from .webkit_indexeddb import WebkitIndexedDB from .webkit_localstorage import WebkitLocalStorage from .webkit_safariviewservice import WebkitSafariViewService from .webkit_session_resource_log import WebkitSessionResourceLog +from .webkit_resource_load_statistics import WebkitResourceLoadStatistics from .chrome_history import ChromeHistory from .chrome_favicon import ChromeFavicon from .firefox_history import FirefoxHistory @@ -31,12 +32,13 @@ from .filesystem import Filesystem BACKUP_MODULES = [SafariBrowserState, SafariHistory, Datausage, SMS, SMSAttachments, ChromeHistory, ChromeFavicon, WebkitSessionResourceLog, - Calls, IDStatusCache, LocationdClients, InteractionC, - FirefoxHistory, FirefoxFavicon, Contacts, Manifest, Whatsapp] + WebkitResourceLoadStatistics, Calls, IDStatusCache, LocationdClients, + InteractionC, FirefoxHistory, FirefoxFavicon, Contacts, Manifest, Whatsapp] FS_MODULES = [IOSVersionHistory, SafariHistory, SafariFavicon, SafariBrowserState, WebkitIndexedDB, WebkitLocalStorage, WebkitSafariViewService, - WebkitSessionResourceLog, Datausage, Netusage, ChromeHistory, + WebkitResourceLoadStatistics, WebkitSessionResourceLog, + Datausage, Netusage, ChromeHistory, ChromeFavicon, Calls, IDStatusCache, SMS, SMSAttachments, LocationdClients, InteractionC, FirefoxHistory, FirefoxFavicon, Contacts, CacheFiles, Whatsapp, Filesystem] diff --git a/mvt/ios/modules/fs/base.py b/mvt/ios/modules/fs/base.py index e90dbfc..c0dcee3 100644 --- a/mvt/ios/modules/fs/base.py +++ b/mvt/ios/modules/fs/base.py @@ -20,6 +20,22 @@ class IOSExtraction(MVTModule): is_fs_dump = False is_sysdiagnose = False + def _is_database_malformed(self, file_path): + # Check if the database is malformed. + conn = sqlite3.connect(file_path) + cur = conn.cursor() + + try: + recover = False + cur.execute("SELECT name FROM sqlite_master WHERE type='table';") + except sqlite3.DatabaseError as e: + if "database disk image is malformed" in str(e): + recover = True + finally: + conn.close() + + return recover + def _recover_database(self, file_path): """Tries to recover a malformed database by running a .clone command. :param file_path: Path to the malformed database file. @@ -49,6 +65,7 @@ class IOSExtraction(MVTModule): """Try to locate the module's database file from either an iTunes backup or a full filesystem dump. :param backup_id: iTunes backup database file's ID (or hash). + :param root_paths: Glob patterns for files to seek in filesystem dump. """ file_path = None # First we check if the was an explicit file path specified. @@ -84,18 +101,5 @@ class IOSExtraction(MVTModule): else: raise DatabaseNotFoundError("Unable to find the module's database file") - # Check if the database is corrupted. - conn = sqlite3.connect(self.file_path) - cur = conn.cursor() - - try: - recover = False - cur.execute("SELECT name FROM sqlite_master WHERE type='table';") - except sqlite3.DatabaseError as e: - if "database disk image is malformed" in str(e): - recover = True - finally: - conn.close() - - if recover: + if self._is_database_malformed(self.file_path): self._recover_database(self.file_path) diff --git a/mvt/ios/modules/fs/webkit_resource_load_statistics.py b/mvt/ios/modules/fs/webkit_resource_load_statistics.py new file mode 100644 index 0000000..2c15c65 --- /dev/null +++ b/mvt/ios/modules/fs/webkit_resource_load_statistics.py @@ -0,0 +1,98 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021 MVT Project Developers. +# See the file 'LICENSE' for usage and copying permissions, or find a copy at +# https://github.com/mvt-project/mvt/blob/main/LICENSE + +import os +import sqlite3 +import datetime + +from .base import IOSExtraction + +from mvt.common.utils import convert_mactime_to_unix, convert_timestamp_to_iso + +WEBKIT_RESOURCELOADSTATICS_BACKUP_RELPATH = "Library/WebKit/WebsiteData/ResourceLoadStatistics/observations.db" +WEBKIT_RESOURCELOADSTATICS_ROOT_PATHS = [ + "private/var/mobile/Containers/Data/Application/*/Library/WebKit/WebsiteData/ResourceLoadStatistics/observations.db", + "private/var/mobile/Containers/Data/Application/*/SystemData/com.apple.SafariViewService/Library/WebKit/WebsiteData/observations.db", +] + +class WebkitResourceLoadStatistics(IOSExtraction): + """This module extracts records from WebKit ResourceLoadStatistics observations.db. + """ + # TODO: Add serialize(). + + def __init__(self, file_path=None, base_folder=None, output_folder=None, + fast_mode=False, log=None, results=[]): + super().__init__(file_path=file_path, base_folder=base_folder, + output_folder=output_folder, fast_mode=fast_mode, + log=log, results=results) + + def check_indicators(self): + if not self.indicators: + return + + self.detected = {} + for key, items in self.results.items(): + for item in items: + if self.indicators.check_domain(item["registrable_domain"]): + if key not in self.detected: + self.detected[key] = [item,] + else: + self.detected[key].append(item) + + def _process_observations_db(self, db_path, key): + self.log.info("Found WebKit ResourceLoadStatistics observations.db file at path %s", db_path) + + if self._is_database_malformed(db_path): + self._recover_database(db_path) + + conn = sqlite3.connect(db_path) + cur = conn.cursor() + + try: + cur.execute("SELECT * from ObservedDomains;") + except sqlite3.OperationalError: + return + + if not key in self.results: + self.results[key] = [] + + for row in cur: + self.results[key].append(dict( + domain_id=row[0], + registrable_domain=row[1], + last_seen=row[2], + had_user_interaction=bool(row[3]), + # TODO: Fix isodate. + last_seen_isodate=convert_timestamp_to_iso(datetime.datetime.utcfromtimestamp(int(row[2]))), + )) + + if len(self.results[key]) > 0: + self.log.info("Extracted a total of %d records from %s", len(self.results[key]), db_path) + + def run(self): + self.results = {} + + if self.is_backup: + manifest_db_path = os.path.join(self.base_folder, "Manifest.db") + if not os.path.exists(manifest_db_path): + self.log.info("Unable to search for WebKit observations.db files in backup because of missing Manifest.db") + return + + try: + conn = sqlite3.connect(manifest_db_path) + cur = conn.cursor() + cur.execute("SELECT fileID, domain FROM Files WHERE relativePath = ?;", (WEBKIT_RESOURCELOADSTATICS_BACKUP_RELPATH,)) + except Exception as e: + self.log.error("Unable to search for WebKit observations.db files in backup because of failed query to Manifest.db: %s", e) + + for row in cur: + file_id = row[0] + domain = row[1] + db_path = os.path.join(self.base_folder, file_id[0:2], file_id) + if os.path.exists(db_path): + self._process_observations_db(db_path=db_path, key=f"{domain}/{WEBKIT_RESOURCELOADSTATICS_BACKUP_RELPATH}") + elif self.is_fs_dump: + for db_path in self._find_paths(WEBKIT_RESOURCELOADSTATICS_ROOT_PATHS): + self._process_observations_db(db_path=db_path, key=os.path.relpath(db_path, self.base_folder))