Merge branch 'main' into tests

docs/iocs.md

@@ -41,6 +41,6 @@ export MVT_STIX2="/home/user/IOC1.stix2:/home/user/IOC2.stix2"
     - [Predator from Cytrox](https://citizenlab.ca/2021/12/pegasus-vs-predator-dissidents-doubly-infected-iphone-reveals-cytrox-mercenary-spyware/) ([STIX2](https://raw.githubusercontent.com/AmnestyTech/investigations/master/2021-12-16_cytrox/cytrox.stix2))
 - [This repository](https://github.com/Te-k/stalkerware-indicators) contains IOCs for Android stalkerware including [a STIX MVT-compatible file](https://raw.githubusercontent.com/Te-k/stalkerware-indicators/master/stalkerware.stix2).
 
-You can automaticallly download the latest public indicator files with the command `mvt-ios download-indicators` or `mvt-android download-indicators`.
+You can automaticallly download the latest public indicator files with the command `mvt-ios download-iocs` or `mvt-android download-iocs`.
 
 Please [open an issue](https://github.com/mvt-project/mvt/issues/) to suggest new sources of STIX-formatted IOCs.

mvt/android/cli.py

@@ -9,11 +9,10 @@ import os
 import click
 from rich.logging import RichHandler
 
-from mvt.common.help import HELP_MSG_MODULE, HELP_MSG_IOC
+from mvt.common.help import (HELP_MSG_FAST, HELP_MSG_IOC,
-from mvt.common.help import HELP_MSG_FAST, HELP_MSG_OUTPUT, HELP_MSG_LIST_MODULES
+                             HELP_MSG_LIST_MODULES, HELP_MSG_MODULE,
-from mvt.common.help import HELP_MSG_SERIAL
+                             HELP_MSG_OUTPUT, HELP_MSG_SERIAL)
-from mvt.common.indicators import Indicators
+from mvt.common.indicators import Indicators, download_indicators_files
-from mvt.common.indicators import download_indicators_files
 from mvt.common.logo import logo
 from mvt.common.module import run_module, save_timeline
 
@@ -199,9 +198,10 @@ def check_backup(ctx, iocs, output, backup_path, serial):
 
         run_module(m)
 
+
 #==============================================================================
-# Command: download-indicators
+# Command: download-iocs
 #==============================================================================
-@cli.command("download-indicators", help="Download public STIX2 indicators")
+@cli.command("download-iocs", help="Download public STIX2 indicators")
 def download_indicators():
     download_indicators_files(log)

mvt/android/modules/adb/__init__.py

@@ -4,6 +4,7 @@
 #   https://license.mvt.re/1.1/
 
 from .chrome_history import ChromeHistory
+from .dumpsys_accessibility import DumpsysAccessibility
 from .dumpsys_batterystats import DumpsysBatterystats
 from .dumpsys_full import DumpsysFull
 from .dumpsys_packages import DumpsysPackages
@@ -18,6 +19,6 @@ from .sms import SMS
 from .whatsapp import Whatsapp
 
 ADB_MODULES = [ChromeHistory, SMS, Whatsapp, Processes,
-               DumpsysBatterystats, DumpsysProcstats,
+               DumpsysAccessibility, DumpsysBatterystats, DumpsysProcstats,
                DumpsysPackages, DumpsysReceivers, DumpsysFull,
                Packages, RootBinaries, Logcat, Files]

mvt/android/modules/adb/dumpsys_accessibility.py

@@ -0,0 +1,53 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021 The MVT Project Authors.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+
+import io
+import logging
+import os
+
+from .base import AndroidExtraction
+
+log = logging.getLogger(__name__)
+
+
+class DumpsysAccessibility(AndroidExtraction):
+    """This module extracts stats on accessibility."""
+
+    def __init__(self, file_path=None, base_folder=None, output_folder=None,
+                 serial=None, fast_mode=False, log=None, results=[]):
+        super().__init__(file_path=file_path, base_folder=base_folder,
+                         output_folder=output_folder, fast_mode=fast_mode,
+                         log=log, results=results)
+
+    def run(self):
+        self._adb_connect()
+
+        stats = self._adb_command("dumpsys accessibility")
+
+        in_services = False
+        for line in stats.split("\n"):
+            if line.strip().startswith("installed services:"):
+                in_services = True
+                continue
+
+            if not in_services:
+                continue
+
+            if line.strip() == "}":
+                break
+
+            service = line.split(":")[1].strip()
+            log.info("Found installed accessibility service \"%s\"", service)
+
+        if self.output_folder:
+            acc_path = os.path.join(self.output_folder,
+                                      "dumpsys_accessibility.txt")
+            with io.open(acc_path, "w", encoding="utf-8") as handle:
+                handle.write(stats)
+
+            log.info("Records from dumpsys accessibility stored at %s",
+                     acc_path)
+
+        self._adb_disconnect()

mvt/android/modules/adb/files.py

@@ -3,12 +3,11 @@
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
-import logging
-import os
-import stat
 import datetime
+import logging
+import stat
 
-from mvt.common.utils import check_for_links, convert_timestamp_to_iso
+from mvt.common.utils import convert_timestamp_to_iso
 
 from .base import AndroidExtraction
 
@@ -31,8 +30,8 @@ class Files(AndroidExtraction):
         # Run find command with correct args and parse results.
 
         # Check that full file printf options are suppported on first run.
-        if self.full_find == None:
+        if self.full_find is None:
-            output = self._adb_command(f"find '/' -maxdepth 1 -printf '%T@ %m %s %u %g %p\n' 2> /dev/null")
+            output = self._adb_command("find '/' -maxdepth 1 -printf '%T@ %m %s %u %g %p\n' 2> /dev/null")
             if not (output or output.strip().splitlines()):
                 # Full  find command failed to generate output, fallback to basic file arguments
                 self.full_find = False
@@ -40,7 +39,7 @@ class Files(AndroidExtraction):
                 self.full_find = True
 
         found_files = []
-        if self.full_find == True:
+        if self.full_find is True:
             # Run full file command and collect additonal file information.
             output = self._adb_command(f"find '{file_path}' -printf '%T@ %m %s %u %g %p\n' 2> /dev/null")
             for file_line in output.splitlines():
@@ -90,7 +89,7 @@ class Files(AndroidExtraction):
             return
 
         for result in self.results:
-            if self.indicators.check_filename(result["path"]):
+            if self.indicators.check_file_name(result["path"]):
                 self.log.warning("Found a known suspicous filename at path: \"%s\"", result["path"])
                 self.detected.append(result)
 

mvt/common/indicators.py

@@ -6,6 +6,7 @@
 import io
 import json
 import os
+
 import requests
 from appdirs import user_data_dir
 
@@ -44,7 +45,7 @@ class Indicators:
 
     def _check_stix2_env_variable(self):
         """
-        Checks if a variable MVT_STIX2 contains path to STIX Files
+        Checks if a variable MVT_STIX2 contains path to STIX Files.
         """
         if "MVT_STIX2" not in os.environ:
             return False
@@ -58,7 +59,7 @@ class Indicators:
 
     def load_indicators_files(self, files, load_default=True):
         """
-        Load a list of indicators files
+        Load a list of indicators files.
         """
         for file_path in files:
             if os.path.isfile(file_path):
@@ -271,7 +272,7 @@ class Indicators:
 
         return False
 
-    def check_filename(self, file_path) -> bool:
+    def check_file_name(self, file_path) -> bool:
         """Check the provided file path against the list of file indicators.
 
         :param file_path: File path or file name to check against file
@@ -290,6 +291,9 @@ class Indicators:
 
         return False
 
+    # TODO: The difference between check_file_name() and check_file_path()
+    #       needs to be more explicit and clear. Probably, the two should just
+    #       be combined into one function.
     def check_file_path(self, file_path) -> bool:
         """Check the provided file path against the list of file indicators.
 
@@ -307,6 +311,7 @@ class Indicators:
             # Strip any trailing slash from indicator paths to match directories.
             if file_path.startswith(ioc_file.rstrip("/")):
                 return True
+
         return False
 
     def check_profile(self, profile_uuid) -> bool:
@@ -326,7 +331,7 @@ class Indicators:
 
 def download_indicators_files(log):
     """
-    Download indicators from repo into MVT app data directory
+    Download indicators from repo into MVT app data directory.
     """
     data_dir = user_data_dir("mvt")
     if not os.path.isdir(data_dir):

mvt/common/version.py

@@ -6,7 +6,7 @@
 import requests
 from packaging import version
 
-MVT_VERSION = "1.4.1"
+MVT_VERSION = "1.4.3"
 
 
 def check_for_updates():

mvt/ios/cli.py

@@ -5,17 +5,15 @@
 
 import logging
 import os
-import io
 
 import click
 from rich.logging import RichHandler
 from rich.prompt import Prompt
 
-from mvt.common.help import HELP_MSG_MODULE, HELP_MSG_IOC
+from mvt.common.help import (HELP_MSG_FAST, HELP_MSG_IOC,
-from mvt.common.help import HELP_MSG_FAST, HELP_MSG_OUTPUT
+                             HELP_MSG_LIST_MODULES, HELP_MSG_MODULE,
-from mvt.common.help import HELP_MSG_LIST_MODULES
+                             HELP_MSG_OUTPUT)
-from mvt.common.indicators import Indicators
+from mvt.common.indicators import Indicators, download_indicators_files
-from mvt.common.indicators import download_indicators_files
 from mvt.common.logo import logo
 from mvt.common.module import run_module, save_timeline
 from mvt.common.options import MutuallyExclusiveOption
@@ -295,9 +293,10 @@ def check_iocs(ctx, iocs, list_modules, module, folder):
             except NotImplementedError:
                 continue
 
+
 #==============================================================================
-# Command: download-indicators
+# Command: download-iocs
 #==============================================================================
-@cli.command("download-indicators", help="Download public STIX2 indicators")
+@cli.command("download-iocs", help="Download public STIX2 indicators")
-def download_indicators():
+def download_iocs():
     download_indicators_files(log)

mvt/ios/modules/backup/configuration_profiles.py

@@ -7,6 +7,7 @@
 import os
 import plistlib
 from base64 import b64encode
+
 from mvt.common.utils import convert_timestamp_to_iso
 
 from ..base import IOSExtraction
@@ -70,7 +71,7 @@ class ConfigurationProfiles(IOSExtraction):
             with open(conf_file_path, "rb") as handle:
                 try:
                     conf_plist = plistlib.load(handle)
-                except:
+                except Exception:
                     conf_plist = {}
 
             if "SignerCerts" in conf_plist:

mvt/ios/modules/backup/manifest.py

@@ -83,7 +83,7 @@ class Manifest(IOSExtraction):
                     self.detected.append(result)
                     continue
 
-            if self.indicators.check_filename(result["relative_path"]):
+            if self.indicators.check_file_name(result["relative_path"]):
                 self.log.warning("Found a known malicious file at path: %s", result["relative_path"])
                 self.detected.append(result)
                 continue

mvt/ios/modules/mixed/__init__.py

@@ -16,13 +16,13 @@ from .net_datausage import Datausage
 from .osanalytics_addaily import OSAnalyticsADDaily
 from .safari_browserstate import SafariBrowserState
 from .safari_history import SafariHistory
+from .shortcuts import Shortcuts
 from .sms import SMS
 from .sms_attachments import SMSAttachments
 from .tcc import TCC
 from .webkit_resource_load_statistics import WebkitResourceLoadStatistics
 from .webkit_session_resource_log import WebkitSessionResourceLog
 from .whatsapp import Whatsapp
-from .shortcuts import Shortcuts
 
 MIXED_MODULES = [Calls, ChromeFavicon, ChromeHistory, Contacts, FirefoxFavicon,
                  FirefoxHistory, IDStatusCache, InteractionC, LocationdClients,

mvt/ios/modules/mixed/shortcuts.py

@@ -3,12 +3,13 @@
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
-import sqlite3
 import io
-import plistlib
 import itertools
+import plistlib
+import sqlite3
 
-from mvt.common.utils import check_for_links, convert_mactime_to_unix, convert_timestamp_to_iso
+from mvt.common.utils import (check_for_links, convert_mactime_to_unix,
+                              convert_timestamp_to_iso)
 
 from ..base import IOSExtraction
 
@@ -33,13 +34,21 @@ class Shortcuts(IOSExtraction):
         found_urls = ""
         if record["action_urls"]:
             found_urls = "- URLs in actions: {}".format(", ".join(record["action_urls"]))
+        desc = ""
+        if record["description"]:
+            desc = record["description"].decode('utf-8', errors='ignore')
 
-        return {
+        return [{
             "timestamp": record["isodate"],
             "module": self.__class__.__name__,
-            "event": "shortcut",
+            "event": "shortcut_created",
-            "data": f"iOS Shortcut '{record['shortcut_name']}': {record['description']} {found_urls}"
+            "data": f"iOS Shortcut '{record['shortcut_name'].decode('utf-8')}': {desc} {found_urls}"
-        }
+        }, {
+            "timestamp": record["modified_date"],
+            "module": self.__class__.__name__,
+            "event": "shortcut_modified",
+            "data": f"iOS Shortcut '{record['shortcut_name'].decode('utf-8')}': {desc} {found_urls}"
+        }]
 
     def check_indicators(self):
         if not self.indicators:
@@ -57,6 +66,7 @@ class Shortcuts(IOSExtraction):
         conn = sqlite3.connect(self.file_path)
         conn.text_factory = bytes
         cur = conn.cursor()
+        try:
             cur.execute("""
                 SELECT
                     ZSHORTCUT.Z_PK as "shortcut_id",
@@ -68,6 +78,13 @@ class Shortcuts(IOSExtraction):
                 FROM ZSHORTCUT
                 LEFT JOIN ZSHORTCUTACTIONS ON ZSHORTCUTACTIONS.ZSHORTCUT == ZSHORTCUT.Z_PK;
             """)
+        except sqlite3.OperationalError:
+            # Table ZSHORTCUT does not exist
+            self.log.info("Invalid shortcut database format, skipping...")
+            cur.close()
+            conn.close()
+            return
+
         names = [description[0] for description in cur.description]
 
         for item in cur:
@@ -90,7 +107,6 @@ class Shortcuts(IOSExtraction):
                 action["urls"] = [url.rstrip("',") for url in extracted_urls]
                 actions.append(action)
 
-            # pprint.pprint(actions)
             shortcut["isodate"] = convert_timestamp_to_iso(convert_mactime_to_unix(shortcut.pop("created_date")))
             shortcut["modified_date"] = convert_timestamp_to_iso(convert_mactime_to_unix(shortcut["modified_date"]))
             shortcut["parsed_actions"] = len(actions)

mvt/ios/modules/mixed/webkit_resource_load_statistics.py

@@ -77,6 +77,7 @@ class WebkitResourceLoadStatistics(IOSExtraction):
                 for backup_file in self._get_backup_files_from_manifest(relative_path=WEBKIT_RESOURCELOADSTATICS_BACKUP_RELPATH):
                     db_path = self._get_backup_file_from_id(backup_file["file_id"])
                     key = f"{backup_file['domain']}/{WEBKIT_RESOURCELOADSTATICS_BACKUP_RELPATH}"
+                    if db_path:
                         self._process_observations_db(db_path=db_path, key=key)
             except Exception as e:
                 self.log.info("Unable to search for WebKit observations.db: %s", e)

mvt/ios/versions.py

@@ -234,6 +234,8 @@ IPHONE_IOS_VERSIONS = [
     {"build": "19A404", "version": "15.0.2"},
     {"build": "19B74", "version": "15.1"},
     {"build": "19B81", "version": "15.1.1"},
+    {"build": "19C56", "version": "15.2"},
+    {"build": "19C63", "version": "15.2.1"},
 ]