crt0mega/mvt
1
0
Fork 0
mirror of https://github.com/mvt-project/mvt.git synced 2024-06-02 03:05:30 +00:00
Code Issues Projects Releases Wiki Activity

Merge branch 'mvt-project:main' into extract-key

Browse Source
This commit is contained in:
Pavel Kirkovsky 2021-07-27 17:15:37 -07:00 committed by GitHub
commit f4340bd4f9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 306 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
.github/workflows/lint-python.yml vendored Normal file
View File

@ -0,0 +1,21 @@
name: Python Lint
on: [pull_request]
jobs:
lint_python:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions/setup-python@v2
- run: pip install bandit black codespell flake8 isort mypy pytest pyupgrade safety
- run: bandit --recursive --skip B108,B112,B404,B602 .
- run: black --check . || true
- run: codespell
- run: flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
- run: flake8 . --count --exit-zero --max-complexity=10 --max-line-length=88 --show-source --statistics
- run: isort --check-only --profile black . || true
- run: pip install -r requirements.txt || true
- run: mypy --install-types --non-interactive . || true
- run: pytest . || true
- run: pytest --doctest-modules . || true
- run: shopt -s globstar && pyupgrade --py36-plus **/*.py || true
- run: safety check

65
Dockerfile Normal file
View File

@ -0,0 +1,65 @@
FROM ubuntu:20.04
# Ref. https://github.com/mvt-project/mvt
# Fixing major OS dependencies
# ----------------------------
RUN apt update \
&& apt install -y python3 python3-pip libusb-1.0-0-dev \
&& apt install -y wget \
&& apt install -y adb \
&& DEBIAN_FRONTEND=noninteractive apt-get -y install default-jre-headless
# Install build tools for libimobiledevice
# ----------------------------------------
RUN apt install -y build-essential \
checkinstall \
git \
autoconf \
automake \
libtool-bin \
libplist-dev \
libusbmuxd-dev \
libssl-dev \
pkg-config
# Clean up
# --------
RUN apt-get clean \
&& rm -rf /var/lib/apt/lists/*
# Build libimobiledevice
# ----------------------
RUN git clone https://github.com/libimobiledevice/libplist
RUN git clone https://github.com/libimobiledevice/libusbmuxd
RUN git clone https://github.com/libimobiledevice/libimobiledevice
RUN git clone https://github.com/libimobiledevice/usbmuxd
RUN cd libplist && ./autogen.sh && make && make install && ldconfig
RUN cd libusbmuxd && PKG_CONFIG_PATH=/usr/local/lib/pkgconfig ./autogen.sh && make && make install && ldconfig
RUN cd libimobiledevice && PKG_CONFIG_PATH=/usr/local/lib/pkgconfig ./autogen.sh --enable-debug && make && make install && ldconfig
RUN cd usbmuxd && PKG_CONFIG_PATH=/usr/local/lib/pkgconfig ./autogen.sh --prefix=/usr --sysconfdir=/etc --localstatedir=/var --runstatedir=/run && make && make install
# Installing MVT
# --------------
RUN pip3 install mvt
# Installing ABE
# --------------
RUN mkdir /opt/abe
RUN wget https://github.com/nelenkov/android-backup-extractor/releases/download/20210709062403-4c55371/abe.jar -O /opt/abe/abe.jar
# Create alias for abe
RUN echo 'alias abe="java -jar /opt/abe/abe.jar"' >> ~/.bashrc
# Setup investigations environment
# --------------------------------
RUN mkdir /home/cases
WORKDIR /home/cases
RUN echo 'echo "Mobile Verification Toolkit @ Docker\n------------------------------------\n\nYou can find information about how to use this image for Android (https://github.com/mvt-project/mvt/tree/master/docs/android) and iOS (https://github.com/mvt-project/mvt/tree/master/docs/ios) in the official docs of the project.\n"' >> ~/.bashrc
RUN echo 'echo "Note that to perform the debug via USB you might need to give the Docker image access to the USB using \"docker run -it --privileged -v /dev/bus/usb:/dev/bus/usb mvt\" or, preferably, the \"--device=\" parameter.\n"' >> ~/.bashrc
CMD /bin/bash

16
LICENSE
View File

@ -1,4 +1,4 @@
MVT License 1.0
MVT License 1.1
===============
1. Definitions
@ -95,10 +95,10 @@ MVT License 1.0
for subsequent analysis.
1.17. "Data Owner" (or "Data Owners")
means an individual or group of individuals who made use of the
electronic device from which Data that is extracted and/or analyzed
originated. "Data Owner" might or might not differ from "Device
Owner".
means an individual or group of individuals who made legitimate use
of the electronic device from which Data that is extracted and/or
analyzed originated. "Data Owner" might or might not differ from
"Device Owner".
2. License Grants and Conditions
--------------------------------
@ -381,8 +381,8 @@ Exhibit A - Source Code Form License Notice
-------------------------------------------
This Source Code Form is subject to the terms of the MVT License,
v. 1.0. If a copy of the MVT License was not distributed with this
file, You can obtain one at TODO.
v. 1.1. If a copy of the MVT License was not distributed with this
file, You can obtain one at https://license.mvt.re/1.1/.
If it is not possible or desirable to put the notice in a particular
file, then You may include the notice in a location (such as a LICENSE
@ -395,7 +395,7 @@ Exhibit B - "Incompatible With Secondary Licenses" Notice
---------------------------------------------------------
This Source Code Form is "Incompatible With Secondary Licenses", as
defined by the MVT License, v. 1.0.
defined by the MVT License, v. 1.1.
This license is an adaption of Mozilla Public License, v. 2.0.

22
README.md
View File

@ -5,25 +5,26 @@
# Mobile Verification Toolkit
[![](https://img.shields.io/pypi/v/mvt)](https://pypi.org/project/mvt/)
[![](https://img.shields.io/badge/docs-blue.svg)](https://mvt.readthedocs.io)
Mobile Verification Toolkit (MVT) is a collection of utilities to simplify and automate the process of gathering forensic traces helpful to identify a potential compromise of Android and iOS devices.
It has been developed and released by the [Amnesty International Security Lab](https://www.amnesty.org/en/tech/) in July 2021 in the context of the [Pegasus project](https://forbiddenstories.org/about-the-pegasus-project/) along with [a technical forensic methodology and forensic evidences](https://www.amnesty.org/en/latest/research/2021/07/forensic-methodology-report-how-to-catch-nso-groups-pegasus/).
It has been developed and released by the [Amnesty International Security Lab](https://www.amnesty.org/en/tech/) in July 2021 in the context of the [Pegasus project](https://forbiddenstories.org/about-the-pegasus-project/) along with [a technical forensic methodology and forensic evidence](https://www.amnesty.org/en/latest/research/2021/07/forensic-methodology-report-how-to-catch-nso-groups-pegasus/).
*Warning*: this tool has been released as a forensic tool for a technical audience. Using it requires some technical skills such as understanding basics of forensic analysis and using command line tools.
[Please check out the documentation.](https://mvt.readthedocs.io/en/latest/)
*Warning*: MVT is a forensic research tool intended for technologists and investigators. Using it requires understanding the basics of forensic analysis and using command-line tools. This is not intended for end-user self-assessment. If you are concerned with the security of your device please seek expert assistance.
## Installation
First you need to install dependencies, on Linux `sudo apt install python3 python3-pip libusb-1.0-0` or on MacOS `brew install python3 libusb`.
MVT can be installed from sources or conveniently using:
Then you can install mvt from pypi with `pip3 install mvt`, or directly from sources:
```bash
git clone https://github.com/mvt-project/mvt.git
cd mvt
pip3 install .
```
pip3 install mvt
```
You will need some dependencies, so please check the [documentation](https://mvt.readthedocs.io/en/latest/install.html).
Alternatively, you can decide to run MVT and all relevant tools through a [Docker container](https://mvt.readthedocs.io/en/latest/docker.html).
## Usage
@ -41,6 +42,7 @@ MVT provides two commands `mvt-ios` and `mvt-android` with the following subcomm
Check out [the documentation to see how to use them](https://mvt.readthedocs.io/en/latest/).
## License
The purpose of MVT is to facilitate the ***consensual forensic analysis*** of devices of those who might be targets of sophisticated mobile spyware attacks, especially members of civil society and marginalized communities. We do not want MVT to enable privacy violations of non-consenting individuals. Therefore, the goal of this license is to prohibit the use of MVT (and any other software licensed the same) for the purpose of *adversarial forensics*.

35
docs/docker.md Normal file
View File

@ -0,0 +1,35 @@
## Using Docker
Using Docker simplifies having all the required dependencies and tools (including most recent versions of [libimobiledevice](https://libimobiledevice.org)) readily installed.
Install Docker following the [official documentation](https://docs.docker.com/get-docker/).
Once installed, you can clone MVT's repository and build its Docker image:
```bash
git clone https://github.com/mvt-project/mvt.git
cd mvt
docker build -t mvt .
```
Test if the image was created successfully:
```bash
docker run -it mvt
```
If a prompt is spawned successfully, you can close it with `exit`.
If you wish to use MVT to test an Android device you will need to enable the container's access to the host's USB devices. You can do so by enabling the `--privileged` flag and mounting the USB bus device as a volume:
```bash
docker run -it --privileged -v /dev/bus/usb:/dev/bus/usb mvt
```
**Please note:** the `--privileged` parameter is generally regarded as a security risk. If you want to learn more about this check out [this explainer on container escapes](https://blog.trailofbits.com/2019/07/19/understanding-docker-container-escapes/) as it gives access to the whole system.
Recent versions of Docker provide a `--device` parameter allowing to specify a precise USB device without enabling `--privileged`:
```bash
docker run -it --device=/dev/<your_usb_port> mvt
```

4
docs/install.md
View File

@ -7,7 +7,7 @@ Before proceeding, please note that mvt requires Python 3.6+ to run. While it sh
First install some basic dependencies that will be necessary to build all required tools:
```bash
sudo apt install python3 python3-pip libusb-1.0-0
sudo apt install python3 python3-pip libusb-1.0-0 sqlite3
```
*libusb-1.0-0* is not required if you intend to only use `mvt-ios` and not `mvt-android`.
@ -19,7 +19,7 @@ Running MVT on Mac requires Xcode and [homebrew](https://brew.sh) to be installe
In order to install dependencies use:
```bash
brew install python3 libusb
brew install python3 libusb sqlite3
```
*libusb* is not required if you intend to only use `mvt-ios` and not `mvt-android`.

2
docs/introduction.md
View File

@ -14,4 +14,4 @@ Mobile Verification Toolkit (MVT) is a collection of utilities designed to facil
While MVT is capable of extracting and processing various types of very personal records typically found on a mobile phone (such as calls history, SMS and WhatsApp messages, etc.), this is intended to help identify potential attack vectors such as malicious SMS messages leading to exploitation.
MVT's purpose is not to facilitate adversial forensics of non-consenting individuals' devices. The use of MVT and derivative products to extract and/or analyse data originating from devices used by individuals not consenting to the procedure is explicitly prohibited in the [license](license.md).
MVT's purpose is not to facilitate adversarial forensics of non-consenting individuals' devices. The use of MVT and derivative products to extract and/or analyse data originating from devices used by individuals not consenting to the procedure is explicitly prohibited in the [license](license.md).

1
mkdocs.yml
View File

@ -28,6 +28,7 @@ nav:
- Welcome: "index.md"
- Introduction: "introduction.md"
- Installation: "install.md"
- Using Docker: "docker.md"
- MVT for iOS:
- iOS Forensic Methodology: "ios/methodology.md"
- Install libimobiledevice: "ios/install.md"

37
mvt/android/modules/adb/base.py
View File

@ -4,6 +4,8 @@
# https://github.com/mvt-project/mvt/blob/main/LICENSE
import os
import random
import string
import sys
import time
import logging
@ -96,7 +98,7 @@ class AndroidExtraction(MVTModule):
"""Check if we have a `su` binary on the Android device.
:returns: Boolean indicating whether a `su` binary is present or not
"""
return bool(self._adb_command("[ ! -f /sbin/su ] || echo 1"))
return bool(self._adb_command("command -v su"))
def _adb_root_or_die(self):
"""Check if we have a `su` binary, otherwise raise an Exception.
@ -111,7 +113,7 @@ class AndroidExtraction(MVTModule):
"""
return self._adb_command(f"su -c {command}")
def _adb_download(self, remote_path, local_path, progress_callback=None):
def _adb_download(self, remote_path, local_path, progress_callback=None, retry_root=True):
"""Download a file form the device.
:param remote_path: Path to download from the device
:param local_path: Path to where to locally store the copy of the file
@ -119,6 +121,37 @@ class AndroidExtraction(MVTModule):
"""
try:
self.device.pull(remote_path, local_path, progress_callback)
except AdbCommandFailureException as e:
if retry_root:
self._adb_download_root(remote_path, local_path, progress_callback)
else:
raise Exception(f"Unable to download file {remote_path}: {e}")
def _adb_download_root(self, remote_path, local_path, progress_callback=None):
try:
# Check if we have root, if not raise an Exception.
self._adb_root_or_die()
# We generate a random temporary filename.
tmp_filename = "tmp_" + ''.join(random.choices(string.ascii_uppercase + string.ascii_lowercase + string.digits, k=10))
# We create a temporary local file.
new_remote_path = f"/sdcard/{tmp_filename}"
# We copy the file from the data folder to /sdcard/.
cp = self._adb_command_as_root(f"cp {remote_path} {new_remote_path}")
if cp.startswith("cp: ") and "No such file or directory" in cp:
raise Exception(f"Unable to process file {remote_path}: File not found")
elif cp.startswith("cp: ") and "Permission denied" in cp:
raise Exception(f"Unable to process file {remote_path}: Permission denied")
# We download from /sdcard/ to the local temporary file.
# If it doesn't work now, don't try again (retry_root=False)
self._adb_download(new_remote_path, local_path, retry_root=False)
# Delete the copy on /sdcard/.
self._adb_command(f"rm -rf {new_remote_path}")
except AdbCommandFailureException as e:
raise Exception(f"Unable to download file {remote_path}: {e}")

3
mvt/android/modules/adb/whatsapp.py
View File

@ -6,6 +6,7 @@
import os
import sqlite3
import logging
import base64
from .base import AndroidExtraction
from mvt.common.utils import convert_timestamp_to_iso, check_for_links
@ -69,6 +70,8 @@ class Whatsapp(AndroidExtraction):
# If we find links in the messages or if they are empty we add them to the list.
if check_for_links(message["data"]) or message["data"].strip() == "":
if (message.get('thumb_image') is not None):
message['thumb_image'] = base64.b64encode(message['thumb_image'])
messages.append(message)
cur.close()

19
mvt/common/module.py
View File

@ -13,6 +13,12 @@ import simplejson as json
from .indicators import Indicators
class DatabaseNotFoundError(Exception):
pass
class DatabaseCorruptedError(Exception):
pass
class MVTModule(object):
"""This class provides a base for all extraction modules."""
@ -83,7 +89,11 @@ class MVTModule(object):
results_file_name = f"{name}.json"
results_json_path = os.path.join(self.output_folder, results_file_name)
with open(results_json_path, "w") as handle:
try:
json.dump(self.results, handle, indent=4)
except Exception as e:
self.log.error("Unable to store results of module %s to file %s: %s",
self.__class__.__name__, results_file_name, e)
if self.detected:
detected_file_name = f"{name}_detected.json"
@ -99,6 +109,7 @@ class MVTModule(object):
"""
for result in self.results:
record = self.serialize(result)
if record:
if type(record) == list:
self.timeline.extend(record)
else:
@ -106,6 +117,7 @@ class MVTModule(object):
for detected in self.detected:
record = self.serialize(detected)
if record:
if type(record) == list:
self.timeline_detected.extend(record)
else:
@ -136,8 +148,11 @@ def run_module(module):
except NotImplementedError:
module.log.exception("The run() procedure of module %s was not implemented yet!",
module.__class__.__name__)
except FileNotFoundError as e:
module.log.error("There might be no data to extract by module %s: %s",
except DatabaseNotFoundError as e:
module.log.info("There might be no data to extract by module %s: %s",
module.__class__.__name__, e)
except DatabaseCorruptedError as e:
module.log.error("The %s module database seems to be corrupted and recovery failed: %s",
module.__class__.__name__, e)
except Exception as e:
module.log.exception("Error in running extraction from module %s: %s",

48
mvt/ios/modules/fs/base.py
View File

@ -3,10 +3,15 @@
# See the file 'LICENSE' for usage and copying permissions, or find a copy at
# https://github.com/mvt-project/mvt/blob/main/LICENSE
import io
import os
import glob
import shutil
import sqlite3
import subprocess
from mvt.common.module import MVTModule
from mvt.common.module import DatabaseNotFoundError, DatabaseCorruptedError
class IOSExtraction(MVTModule):
"""This class provides a base for all iOS filesystem/backup extraction modules."""
@ -15,6 +20,31 @@ class IOSExtraction(MVTModule):
is_fs_dump = False
is_sysdiagnose = False
def _recover_database(self, file_path):
"""Tries to recover a malformed database by running a .clone command.
:param file_path: Path to the malformed database file.
"""
# TODO: Find a better solution.
self.log.info("Database at path %s is malformed. Trying to recover...", file_path)
if not os.path.exists(file_path):
return
if not shutil.which("sqlite3"):
raise DatabaseCorruptedError("Unable to recover without sqlite3 binary. Please install sqlite3!")
bak_path = f"{file_path}.bak"
shutil.move(file_path, bak_path)
cmd = f"sqlite3 {bak_path} '.clone {file_path}'"
ret = subprocess.call(cmd, shell=True, stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
if ret != 0:
raise DatabaseCorruptedError("Recovery of database failed")
self.log.info("Database at path %s recovered successfully!", file_path)
def _find_ios_database(self, backup_ids=None, root_paths=[]):
"""Try to locate the module's database file from either an iTunes
backup or a full filesystem dump.
@ -52,4 +82,20 @@ class IOSExtraction(MVTModule):
if file_path:
self.file_path = file_path
else:
raise FileNotFoundError("Unable to find the module's database file")
raise DatabaseNotFoundError("Unable to find the module's database file")
# Check if the database is corrupted.
conn = sqlite3.connect(self.file_path)
cur = conn.cursor()
try:
recover = False
cur.execute("SELECT name FROM sqlite_master WHERE type='table';")
except sqlite3.DatabaseError as e:
if "database disk image is malformed" in str(e):
recover = True
finally:
conn.close()
if recover:
self._recover_database(self.file_path)

2
mvt/ios/modules/fs/locationd.py
View File

@ -62,7 +62,7 @@ class LocationdClients(IOSExtraction):
result["package"] = app
for ts in self.timestamps:
if ts in result.keys():
result[ts] = convert_timestamp_to_iso(convert_mactime_to_unix(result[date]))
result[ts] = convert_timestamp_to_iso(convert_mactime_to_unix(result[ts]))
self.results.append(result)

5
mvt/ios/modules/fs/manifest.py
View File

@ -40,6 +40,8 @@ class Manifest(IOSExtraction):
def serialize(self, record):
records = []
if "modified" not in record or "statusChanged" not in record:
return
for ts in set([record["created"], record["modified"], record["statusChanged"]]):
macb = ""
macb += "M" if ts == record["modified"] else "-"
@ -63,7 +65,10 @@ class Manifest(IOSExtraction):
for result in self.results:
if not "relativePath" in result:
continue
if not result["relativePath"]:
continue
if result["domain"]:
if os.path.basename(result["relativePath"]) == "com.apple.CrashReporter.plist" and result["domain"] == "RootDomain":
self.log.warning("Found a potentially suspicious \"com.apple.CrashReporter.plist\" file created in RootDomain")
self.detected.append(result)

4
mvt/ios/modules/fs/net_base.py
View File

@ -167,6 +167,10 @@ class NetBase(IOSExtraction):
results_by_proc = {proc["proc_id"]: proc for proc in self.results if proc["proc_id"]}
all_proc_id = sorted(results_by_proc.keys())
# Fix issue #108
if not all_proc_id:
return
missing_procs, last_proc_id = {}, None
for proc_id in range(min(all_proc_id), max(all_proc_id)):
if proc_id not in all_proc_id:

1
mvt/ios/versions.py
View File

@ -102,6 +102,7 @@ IPHONE_IOS_VERSIONS = [
{"build": "10A405", "version": "6.0"},
{"build": "11B601", "version": "7.0.5"},
{"build": "18F72", "version": "14.6"},
{"build": "18G69", "version": "14.7"},
{"build": "18E199", "version": "14.5"},
{"build": "18E212", "version": "14.5.1"},
{"build": "18D52", "version": "14.4"},

22
setup.py
View File

@ -7,7 +7,7 @@ import os
from setuptools import setup, find_packages
__package_name__ = "mvt"
__version__ = "1.0.11"
__version__ = "1.0.13"
__description__ = "Mobile Verification Toolkit"
this_directory = os.path.abspath(os.path.dirname(__file__))
@ -17,18 +17,18 @@ with open(readme_path, encoding="utf-8") as handle:
requires = (
# Base dependencies:
"click",
"rich",
"tld",
"tqdm",
"requests",
"simplejson",
"click>=8.0.1",
"rich>=10.6.0",
"tld>=0.12.6",
"tqdm>=4.61.2",
"requests>=2.26.0",
"simplejson>=3.17.3",
# iOS dependencies:
"biplist",
"iOSbackup",
"biplist>=1.0.3",
"iOSbackup>=0.9.912",
# Android dependencies:
"adb-shell",
"libusb1",
"adb-shell>=0.4.0",
"libusb1>=1.9.3",
)
def get_package_data(package):