2021-07-29 11:46:58 +00:00
|
|
|
# Mobile Verification Toolkit (MVT)
|
2023-09-09 15:55:27 +00:00
|
|
|
# Copyright (c) 2021-2023 The MVT Authors.
|
2021-08-01 19:11:08 +00:00
|
|
|
# Use of this software is governed by the MVT License 1.1 that can be found at
|
|
|
|
# https://license.mvt.re/1.1/
|
2021-07-29 11:46:58 +00:00
|
|
|
|
2022-06-17 20:30:46 +00:00
|
|
|
import logging
|
2021-07-29 11:46:58 +00:00
|
|
|
import os
|
|
|
|
import sqlite3
|
2023-01-13 11:58:26 +00:00
|
|
|
from typing import Optional, Union
|
2021-07-29 11:46:58 +00:00
|
|
|
|
2022-08-13 12:04:10 +00:00
|
|
|
from mvt.common.utils import convert_unix_to_iso
|
2021-07-29 11:46:58 +00:00
|
|
|
|
2021-08-15 11:14:18 +00:00
|
|
|
from ..base import IOSExtraction
|
2021-07-30 09:40:09 +00:00
|
|
|
|
2022-08-16 16:26:34 +00:00
|
|
|
WEBKIT_RESOURCELOADSTATICS_BACKUP_RELPATH = "Library/WebKit/WebsiteData/ResourceLoadStatistics/observations.db" # pylint: disable=line-too-long
|
2021-07-29 11:46:58 +00:00
|
|
|
WEBKIT_RESOURCELOADSTATICS_ROOT_PATHS = [
|
2022-08-16 16:26:34 +00:00
|
|
|
"private/var/mobile/Containers/Data/Application/*/Library/WebKit/WebsiteData/ResourceLoadStatistics/observations.db", # pylint: disable=line-too-long
|
|
|
|
"private/var/mobile/Containers/Data/Application/*/SystemData/com.apple.SafariViewService/Library/WebKit/WebsiteData/observations.db", # pylint: disable=line-too-long
|
2021-07-29 11:46:58 +00:00
|
|
|
]
|
|
|
|
|
2021-11-19 14:27:51 +00:00
|
|
|
|
2021-07-29 11:46:58 +00:00
|
|
|
class WebkitResourceLoadStatistics(IOSExtraction):
|
2022-08-13 00:14:24 +00:00
|
|
|
"""This module extracts records from WebKit ResourceLoadStatistics
|
|
|
|
observations.db."""
|
2021-07-29 11:46:58 +00:00
|
|
|
|
2022-08-16 11:39:55 +00:00
|
|
|
def __init__(
|
|
|
|
self,
|
2022-08-17 13:52:17 +00:00
|
|
|
file_path: Optional[str] = None,
|
|
|
|
target_path: Optional[str] = None,
|
|
|
|
results_path: Optional[str] = None,
|
2023-07-17 16:29:43 +00:00
|
|
|
module_options: Optional[dict] = None,
|
2022-08-16 11:39:55 +00:00
|
|
|
log: logging.Logger = logging.getLogger(__name__),
|
2023-06-01 21:40:26 +00:00
|
|
|
results: Optional[list] = None,
|
2022-08-16 11:39:55 +00:00
|
|
|
) -> None:
|
2023-06-01 21:40:26 +00:00
|
|
|
super().__init__(
|
|
|
|
file_path=file_path,
|
|
|
|
target_path=target_path,
|
|
|
|
results_path=results_path,
|
2023-07-17 16:29:43 +00:00
|
|
|
module_options=module_options,
|
2023-06-01 21:40:26 +00:00
|
|
|
log=log,
|
|
|
|
results=results,
|
|
|
|
)
|
2021-07-29 11:46:58 +00:00
|
|
|
|
2023-01-13 11:58:26 +00:00
|
|
|
self.results = [] if not results else results
|
|
|
|
|
|
|
|
def serialize(self, record: dict) -> Union[dict, list]:
|
|
|
|
msg = f"Webkit resource loaded from {record['registrable_domain']}"
|
|
|
|
if record["domain"] != "":
|
|
|
|
msg += f" by app in domain {record['domain']}"
|
|
|
|
return {
|
|
|
|
"timestamp": record["last_seen_isodate"],
|
|
|
|
"module": self.__class__.__name__,
|
|
|
|
"event": "visit",
|
2023-06-01 21:40:26 +00:00
|
|
|
"data": msg,
|
2023-01-13 11:58:26 +00:00
|
|
|
}
|
2021-08-15 16:53:02 +00:00
|
|
|
|
2022-06-17 20:30:46 +00:00
|
|
|
def check_indicators(self) -> None:
|
2021-07-29 11:46:58 +00:00
|
|
|
if not self.indicators:
|
|
|
|
return
|
|
|
|
|
2023-07-18 16:02:42 +00:00
|
|
|
self.detected = []
|
2023-01-13 11:58:26 +00:00
|
|
|
for result in self.results:
|
|
|
|
ioc = self.indicators.check_domain(result["registrable_domain"])
|
|
|
|
if ioc:
|
|
|
|
result["matched_indicator"] = ioc
|
|
|
|
self.detected.append(result)
|
|
|
|
|
|
|
|
def _process_observations_db(self, db_path: str, domain: str, path: str) -> None:
|
2023-06-01 21:40:26 +00:00
|
|
|
self.log.info(
|
|
|
|
"Found WebKit ResourceLoadStatistics observations.db file at path %s",
|
|
|
|
db_path,
|
|
|
|
)
|
2021-07-29 11:46:58 +00:00
|
|
|
|
2021-08-16 08:50:35 +00:00
|
|
|
self._recover_sqlite_db_if_needed(db_path)
|
2021-07-29 11:46:58 +00:00
|
|
|
|
2023-11-27 16:55:44 +00:00
|
|
|
conn = self._open_sqlite_db(db_path)
|
2021-07-29 11:46:58 +00:00
|
|
|
cur = conn.cursor()
|
|
|
|
|
|
|
|
try:
|
2023-06-01 21:40:26 +00:00
|
|
|
# FIXME: table contains extra fields with timestamp here
|
|
|
|
cur.execute(
|
|
|
|
"""
|
2023-04-07 12:43:20 +00:00
|
|
|
SELECT
|
|
|
|
domainID,
|
|
|
|
registrableDomain,
|
|
|
|
lastSeen,
|
|
|
|
hadUserInteraction
|
|
|
|
from ObservedDomains;
|
2023-06-01 21:40:26 +00:00
|
|
|
"""
|
|
|
|
)
|
2021-07-29 11:46:58 +00:00
|
|
|
except sqlite3.OperationalError:
|
|
|
|
return
|
|
|
|
|
|
|
|
for row in cur:
|
2023-06-01 21:40:26 +00:00
|
|
|
self.results.append(
|
|
|
|
{
|
|
|
|
"domain_id": row[0],
|
|
|
|
"registrable_domain": row[1],
|
|
|
|
"last_seen": row[2],
|
|
|
|
"had_user_interaction": bool(row[3]),
|
|
|
|
"last_seen_isodate": convert_unix_to_iso(row[2]),
|
|
|
|
"domain": domain,
|
|
|
|
"path": path,
|
|
|
|
}
|
|
|
|
)
|
2021-07-29 11:46:58 +00:00
|
|
|
|
2023-01-13 11:58:26 +00:00
|
|
|
if len(self.results) > 0:
|
2023-06-01 21:40:26 +00:00
|
|
|
self.log.info(
|
|
|
|
"Extracted a total of %d records from %s", len(self.results), db_path
|
|
|
|
)
|
2021-07-29 11:46:58 +00:00
|
|
|
|
2022-06-17 20:30:46 +00:00
|
|
|
def run(self) -> None:
|
2021-07-29 11:46:58 +00:00
|
|
|
if self.is_backup:
|
|
|
|
try:
|
2022-08-16 11:39:55 +00:00
|
|
|
for backup_file in self._get_backup_files_from_manifest(
|
2023-06-01 21:40:26 +00:00
|
|
|
relative_path=WEBKIT_RESOURCELOADSTATICS_BACKUP_RELPATH
|
|
|
|
):
|
2021-09-01 19:59:12 +00:00
|
|
|
db_path = self._get_backup_file_from_id(backup_file["file_id"])
|
2022-08-16 11:39:55 +00:00
|
|
|
|
2022-01-10 17:09:31 +00:00
|
|
|
if db_path:
|
2023-06-01 21:40:26 +00:00
|
|
|
self._process_observations_db(
|
|
|
|
db_path=db_path,
|
|
|
|
domain=backup_file["domain"],
|
|
|
|
path=WEBKIT_RESOURCELOADSTATICS_BACKUP_RELPATH,
|
|
|
|
)
|
2022-08-13 00:14:24 +00:00
|
|
|
except Exception as exc:
|
2022-08-13 12:04:10 +00:00
|
|
|
self.log.info("Unable to find WebKit observations.db: %s", exc)
|
2021-07-29 11:46:58 +00:00
|
|
|
elif self.is_fs_dump:
|
2022-08-16 11:39:55 +00:00
|
|
|
for db_path in self._get_fs_files_from_patterns(
|
2023-06-01 21:40:26 +00:00
|
|
|
WEBKIT_RESOURCELOADSTATICS_ROOT_PATHS
|
|
|
|
):
|
2022-08-13 12:04:10 +00:00
|
|
|
db_rel_path = os.path.relpath(db_path, self.target_path)
|
2023-06-01 21:40:26 +00:00
|
|
|
self._process_observations_db(
|
|
|
|
db_path=db_path, domain="", path=db_rel_path
|
|
|
|
)
|