diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml new file mode 100644 index 0000000..06d6413 --- /dev/null +++ b/.github/workflows/python-package.yml @@ -0,0 +1,43 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: Python package + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: [3.7, 3.8, 3.9] + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install flake8 pytest safety + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + - name: Lint with flake8 + run: | + # stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + - name: Safety checks + run: safety check + + # - name: Test with pytest + # run: | + # pytest diff --git a/.gitignore b/.gitignore index 73b7def..985254e 100644 --- a/.gitignore +++ b/.gitignore @@ -129,3 +129,5 @@ dmypy.json .pyre/ *.pyc +# Temporal files +*~ diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 0000000..7b28589 --- /dev/null +++ b/AUTHORS @@ -0,0 +1,7 @@ +MVT was originally authored by Claudio Guarnieri . + +For an up-to-date list of all contributors visit: + https://github.com/mvt-project/mvt/graphs/contributors + +Or run: + git shortlog -s -n diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..bb467a6 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,66 @@ +FROM ubuntu:20.04 + +# Ref. https://github.com/mvt-project/mvt + +# Fixing major OS dependencies +# ---------------------------- +RUN apt update \ + && apt install -y python3 python3-pip libusb-1.0-0-dev \ + && apt install -y wget \ + && apt install -y adb \ + && DEBIAN_FRONTEND=noninteractive apt-get -y install default-jre-headless + +# Install build tools for libimobiledevice +# ---------------------------------------- +RUN apt install -y build-essential \ + checkinstall \ + git \ + autoconf \ + automake \ + libtool-bin \ + libplist-dev \ + libusbmuxd-dev \ + libssl-dev \ + sqlite3 \ + pkg-config + +# Clean up +# -------- +RUN apt-get clean \ + && rm -rf /var/lib/apt/lists/* + + +# Build libimobiledevice +# ---------------------- +RUN git clone https://github.com/libimobiledevice/libplist +RUN git clone https://github.com/libimobiledevice/libusbmuxd +RUN git clone https://github.com/libimobiledevice/libimobiledevice +RUN git clone https://github.com/libimobiledevice/usbmuxd + +RUN cd libplist && ./autogen.sh && make && make install && ldconfig + +RUN cd libusbmuxd && PKG_CONFIG_PATH=/usr/local/lib/pkgconfig ./autogen.sh && make && make install && ldconfig + +RUN cd libimobiledevice && PKG_CONFIG_PATH=/usr/local/lib/pkgconfig ./autogen.sh --enable-debug && make && make install && ldconfig + +RUN cd usbmuxd && PKG_CONFIG_PATH=/usr/local/lib/pkgconfig ./autogen.sh --prefix=/usr --sysconfdir=/etc --localstatedir=/var --runstatedir=/run && make && make install + +# Installing MVT +# -------------- +RUN pip3 install mvt + +# Installing ABE +# -------------- +RUN mkdir /opt/abe +RUN wget https://github.com/nelenkov/android-backup-extractor/releases/download/20210709062403-4c55371/abe.jar -O /opt/abe/abe.jar +# Create alias for abe +RUN echo 'alias abe="java -jar /opt/abe/abe.jar"' >> ~/.bashrc + +# Setup investigations environment +# -------------------------------- +RUN mkdir /home/cases +WORKDIR /home/cases +RUN echo 'echo "Mobile Verification Toolkit @ Docker\n------------------------------------\n\nYou can find information about how to use this image for Android (https://github.com/mvt-project/mvt/tree/master/docs/android) and iOS (https://github.com/mvt-project/mvt/tree/master/docs/ios) in the official docs of the project.\n"' >> ~/.bashrc +RUN echo 'echo "Note that to perform the debug via USB you might need to give the Docker image access to the USB using \"docker run -it --privileged -v /dev/bus/usb:/dev/bus/usb mvt\" or, preferably, the \"--device=\" parameter.\n"' >> ~/.bashrc + +CMD /bin/bash diff --git a/LICENSE b/LICENSE index 54123c1..94f1ae4 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -MVT License 1.0 +MVT License 1.1 =============== 1. Definitions @@ -35,7 +35,7 @@ MVT License 1.0 means any form of the work other than Source Code Form. 1.7. "Larger Work" - means a work that combines Covered Software with other material, in + means a work that combines Covered Software with other material, in a separate file or files, that is not Covered Software. 1.8. "License" @@ -89,16 +89,16 @@ MVT License 1.0 a Larger Work. 1.16. "Device Owner" (or "Device Owners") - means an individal or a legal entity with legal ownership of an + means an individual or a legal entity with legal ownership of an electronic device which is being analysed through the use of Covered Software or a Larger Work, or from which Data was extracted for subsequent analysis. 1.17. "Data Owner" (or "Data Owners") - means an individial or group of individuals who made use of the - electronic device from which Data that is extracted and/or analyzed - originated. "Data Owner" might or might not differ from "Device - Owner". + means an individual or group of individuals who made legitimate use + of the electronic device from which Data that is extracted and/or + analyzed originated. "Data Owner" might or might not differ from + "Device Owner". 2. License Grants and Conditions -------------------------------- @@ -381,8 +381,8 @@ Exhibit A - Source Code Form License Notice ------------------------------------------- This Source Code Form is subject to the terms of the MVT License, - v. 1.0. If a copy of the MVT License was not distributed with this - file, You can obtain one at TODO. + v. 1.1. If a copy of the MVT License was not distributed with this + file, You can obtain one at https://license.mvt.re/1.1/. If it is not possible or desirable to put the notice in a particular file, then You may include the notice in a location (such as a LICENSE @@ -395,7 +395,7 @@ Exhibit B - "Incompatible With Secondary Licenses" Notice --------------------------------------------------------- This Source Code Form is "Incompatible With Secondary Licenses", as - defined by the MVT License, v. 1.0. + defined by the MVT License, v. 1.1. This license is an adaption of Mozilla Public License, v. 2.0. diff --git a/README.md b/README.md index ac1e3a8..16964c4 100644 --- a/README.md +++ b/README.md @@ -5,23 +5,27 @@ # Mobile Verification Toolkit [![](https://img.shields.io/pypi/v/mvt)](https://pypi.org/project/mvt/) +[![Documentation Status](https://readthedocs.org/projects/mvt/badge/?version=latest)](https://docs.mvt.re/en/latest/?badge=latest) Mobile Verification Toolkit (MVT) is a collection of utilities to simplify and automate the process of gathering forensic traces helpful to identify a potential compromise of Android and iOS devices. -It has been developed and released by the [Amnesty International Security Lab](https://www.amnesty.org/en/tech/) in July 2021 in the context of the [Pegasus project](https://forbiddenstories.org/about-the-pegasus-project/) along with [a technical forensic methodology and forensic evidences](https://www.amnesty.org/en/latest/research/2021/07/forensic-methodology-report-how-to-catch-nso-groups-pegasus/). +It has been developed and released by the [Amnesty International Security Lab](https://www.amnesty.org/en/tech/) in July 2021 in the context of the [Pegasus project](https://forbiddenstories.org/about-the-pegasus-project/) along with [a technical forensic methodology and forensic evidence](https://www.amnesty.org/en/latest/research/2021/07/forensic-methodology-report-how-to-catch-nso-groups-pegasus/). -[Please check out the documentation.](https://mvt.readthedocs.io/en/latest/) +*Warning*: MVT is a forensic research tool intended for technologists and investigators. Using it requires understanding the basics of forensic analysis and using command-line tools. This is not intended for end-user self-assessment. If you are concerned with the security of your device please seek expert assistance. ## Installation -First you need to install dependencies, on Linux `sudo apt install python3 python3-pip libusb-1.0-0` or on MacOS `brew install python3 libusb`. +MVT can be installed from sources or conveniently using: -Then you can install mvt from pypi with `pip3 install mvt`, or directly from sources: -```bash -git clone https://github.com/mvt-project/mvt.git -cd mvt -pip3 install . ``` +pip3 install mvt +``` + +You will need some dependencies, so please check the [documentation](https://docs.mvt.re/en/latest/install.html). + +Alternatively, you can decide to run MVT and all relevant tools through a [Docker container](https://docs.mvt.re/en/latest/docker.html). + +**Please note:** [MVT does not currently support running natively on Windows.](https://docs.mvt.re/en/latest/install.html#mvt-on-windows) ## Usage @@ -32,11 +36,12 @@ MVT provides two commands `mvt-ios` and `mvt-android` with the following subcomm * `check-fs`: Extract artifacts from a full filesystem dump * `check-iocs`: Compare stored JSON results to provided indicators * `decrypt-backup`: Decrypt an encrypted iTunes backup + * `extract-key`: Extract decryption key from an iTunes backup * `mvt-android`: * `check-backup`: Check an Android Backup * `download-apks`: Download all or non-safelisted installed APKs -Check out [the documentation to see how to use them](https://mvt.readthedocs.io/en/latest/). +Check out [the documentation to see how to use them](https://docs.mvt.re/). ## License diff --git a/dev/mvt-android b/dev/mvt-android index 4ac6a2f..53ae739 100755 --- a/dev/mvt-android +++ b/dev/mvt-android @@ -1,8 +1,8 @@ #!/usr/bin/env python3 # Mobile Verification Toolkit (MVT) -# Copyright (c) 2021 MVT Project Developers. -# See the file 'LICENSE' for usage and copying permissions, or find a copy at -# https://github.com/mvt-project/mvt/blob/main/LICENSE +# Copyright (c) 2021 The MVT Project Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ import os import sys @@ -10,4 +10,5 @@ import sys sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from mvt import android + android.cli() diff --git a/dev/mvt-ios b/dev/mvt-ios index ab89c1e..a01f417 100755 --- a/dev/mvt-ios +++ b/dev/mvt-ios @@ -1,8 +1,8 @@ #!/usr/bin/env python3 # Mobile Verification Toolkit (MVT) -# Copyright (c) 2021 MVT Project Developers. -# See the file 'LICENSE' for usage and copying permissions, or find a copy at -# https://github.com/mvt-project/mvt/blob/main/LICENSE +# Copyright (c) 2021 The MVT Project Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ import os import sys @@ -10,4 +10,5 @@ import sys sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from mvt import ios + ios.cli() diff --git a/docs/android/methodology.md b/docs/android/methodology.md index 2c57488..c064da9 100644 --- a/docs/android/methodology.md +++ b/docs/android/methodology.md @@ -1,8 +1,3 @@ # Methodology for Android forensic -For different technical reasons, it is more complex to do a forensic analysis of an Android phone. - -Currently MVT allows to perform two different checks on an Android phone: - -* Download APKs installed in order to analyze them -* Extract Android backup in order to look for suspicious SMS +TODO diff --git a/docs/docker.md b/docs/docker.md new file mode 100644 index 0000000..bb2d1bf --- /dev/null +++ b/docs/docker.md @@ -0,0 +1,33 @@ +Using Docker simplifies having all the required dependencies and tools (including most recent versions of [libimobiledevice](https://libimobiledevice.org)) readily installed. + +Install Docker following the [official documentation](https://docs.docker.com/get-docker/). + +Once installed, you can clone MVT's repository and build its Docker image: + +```bash +git clone https://github.com/mvt-project/mvt.git +cd mvt +docker build -t mvt . +``` + +Test if the image was created successfully: + +```bash +docker run -it mvt +``` + +If a prompt is spawned successfully, you can close it with `exit`. + +If you wish to use MVT to test an Android device you will need to enable the container's access to the host's USB devices. You can do so by enabling the `--privileged` flag and mounting the USB bus device as a volume: + +```bash +docker run -it --privileged -v /dev/bus/usb:/dev/bus/usb mvt +``` + +**Please note:** the `--privileged` parameter is generally regarded as a security risk. If you want to learn more about this check out [this explainer on container escapes](https://blog.trailofbits.com/2019/07/19/understanding-docker-container-escapes/) as it gives access to the whole system. + +Recent versions of Docker provide a `--device` parameter allowing to specify a precise USB device without enabling `--privileged`: + +```bash +docker run -it --device=/dev/ mvt +``` diff --git a/docs/install.md b/docs/install.md index 5e24576..68d3891 100644 --- a/docs/install.md +++ b/docs/install.md @@ -7,11 +7,13 @@ Before proceeding, please note that mvt requires Python 3.6+ to run. While it sh First install some basic dependencies that will be necessary to build all required tools: ```bash -sudo apt install python3 python3-pip libusb-1.0-0 +sudo apt install python3 python3-pip libusb-1.0-0 sqlite3 ``` *libusb-1.0-0* is not required if you intend to only use `mvt-ios` and not `mvt-android`. +When working with Android devices you should additionally install [Android SDK Platform Tools](https://developer.android.com/studio/releases/platform-tools). If you prefer to install a package made available by your distribution of choice, please make sure the version is recent to ensure compatibility with modern Android devices. + ## Dependencies on Mac Running MVT on Mac requires Xcode and [homebrew](https://brew.sh) to be installed. @@ -19,11 +21,25 @@ Running MVT on Mac requires Xcode and [homebrew](https://brew.sh) to be installe In order to install dependencies use: ```bash -brew install python3 libusb +brew install python3 libusb sqlite3 ``` *libusb* is not required if you intend to only use `mvt-ios` and not `mvt-android`. +When working with Android devices you should additionally install Android SDK Platform Tools: + +```bash +brew install --cask android-platform-tools +``` + +Or by downloading the [official binary releases](https://developer.android.com/studio/releases/platform-tools). + +## MVT on Windows + +MVT does not currently officially support running natively on Windows. While most functionality should work out of the box, there are known issues especially with `mvt-android`. + +It is recommended to try installing and running MVT from [Windows Subsystem Linux (WSL)](https://docs.microsoft.com/en-us/windows/wsl/about) and follow Linux installation instructions for your distribution of choice. + ## Installing MVT If you haven't done so, you can add this to your `.bashrc` or `.zshrc` file in order to add locally installed Pypi binaries to your `$PATH`: @@ -35,7 +51,7 @@ export PATH=$PATH:~/.local/bin Then you can install MVT directly from [pypi](https://pypi.org/project/mvt/) ```bash -pip install mvt +pip3 install mvt ``` Or from the source code: diff --git a/docs/introduction.md b/docs/introduction.md index 06d66ea..740efaf 100644 --- a/docs/introduction.md +++ b/docs/introduction.md @@ -14,4 +14,4 @@ Mobile Verification Toolkit (MVT) is a collection of utilities designed to facil While MVT is capable of extracting and processing various types of very personal records typically found on a mobile phone (such as calls history, SMS and WhatsApp messages, etc.), this is intended to help identify potential attack vectors such as malicious SMS messages leading to exploitation. -MVT's purpose is not to facilitate adversial forensics of non-consenting individuals' devices. The use of MVT and derivative products to extract and/or analyse data originating from devices used by individuals not consenting to the procedure is explicitly prohibited in the [license](license.md). +MVT's purpose is not to facilitate adversarial forensics of non-consenting individuals' devices. The use of MVT and derivative products to extract and/or analyse data originating from devices used by individuals not consenting to the procedure is explicitly prohibited in the [license](license.md). diff --git a/docs/iocs.md b/docs/iocs.md new file mode 100644 index 0000000..5a903ce --- /dev/null +++ b/docs/iocs.md @@ -0,0 +1,32 @@ +# Indicators of Compromise (IOCs) + +MVT uses [Structured Threat Information Expression (STIX)](https://oasis-open.github.io/cti-documentation/stix/intro.html) files to identify potential traces of compromise. + +These indicators of compromise are contained in a file with a particular structure of [JSON](https://en.wikipedia.org/wiki/JSON) with the `.stix2` or `.json` extensions. + +You can indicate a path to a STIX2 indicators file when checking iPhone backups or filesystem dumps. For example: + +```bash +mvt-ios check-backup --iocs ~/ios/malware.stix2 --output /path/to/iphone/output /path/to/backup +``` + +Or, with data from an Android backup: + +```bash +mvt-android check-backup --iocs ~/iocs/malware.stix2 /path/to/android/backup/ +``` + +After extracting forensics data from a device, you are also able to compare it with any STIX2 file you indicate: + +```bash +mvt-ios check-iocs --iocs ~/iocs/malware.stix2 /path/to/iphone/output/ +``` + +If you're looking for indicators of compromise for a specific piece of malware or adversary, please ask investigators or anti-malware researchers who have the relevant expertise for a STIX file. + +## Known repositories of STIX2 IOCs + +- The [Amnesty International investigations repository](https://github.com/AmnestyTech/investigations) contains STIX-formatted IOCs for: + - [Pegasus](https://en.wikipedia.org/wiki/Pegasus_(spyware)) ([STIX2](https://raw.githubusercontent.com/AmnestyTech/investigations/master/2021-07-18_nso/pegasus.stix2)) + +Please [open an issue](https://github.com/mvt-project/mvt/issues/) to suggest new sources of STIX-formatted IOCs. diff --git a/docs/ios/backup/check.md b/docs/ios/backup/check.md index 3a1fd70..b51e074 100644 --- a/docs/ios/backup/check.md +++ b/docs/ios/backup/check.md @@ -2,6 +2,32 @@ The backup might take some time. It is best to make sure the phone remains unlocked during the backup process. Afterwards, a new folder will be created under the path you specified using the UDID of the iPhone you backed up. +## Extracting and saving the decryption key (optional) + +If you do not wish to enter a password every time when decrypting a backup, MVT can accept a key file instead. This key can be used with the `decrypt-backup` command. + +To generate a key file, you will need your device backup and the backup password: + + $ mvt-ios extract-key --help + Usage: mvt-ios extract-key [OPTIONS] BACKUP_PATH + + Extract decryption key from an iTunes backup + + Options: + -p, --password TEXT Password to use to decrypt the backup [required] + -k, --key-file FILE Key file to be written (if unset, will print to STDOUT) + --help Show this message and exit. + +You can specify the password on the command line, or omit the `-p` option to have MVT prompt for a password. The `-k` option specifies where to save the file containing the decryption key. If `-k` is omitted, MVT will display the decryption key without saving. + +_Note_: This decryption key is sensitive data! Keep the file safe. + +To extract the key and have MVT prompt for a password: + +```bash +mvt-ios extract-key -k /path/to/save/key /path/to/backup +``` + ## Decrypting a backup In case you have an encrypted backup, you will need to decrypt it first. This can be done with `mvt-ios` as well: @@ -15,9 +41,10 @@ In case you have an encrypted backup, you will need to decrypt it first. This ca -d, --destination TEXT Path to the folder where to store the decrypted backup [required] - -p, --password TEXT Password to use to decrypt the backup NOTE: This - argument is mutually exclusive with arguments: - [key_file]. + -p, --password TEXT Password to use to decrypt the backup (or, set + MVT_IOS_BACKUP_PASSWORD environment variable) + NOTE: This argument is mutually exclusive with + arguments: [key_file]. -k, --key-file PATH File containing raw encryption key to use to decrypt the backup NOTE: This argument is mutually exclusive @@ -25,10 +52,10 @@ In case you have an encrypted backup, you will need to decrypt it first. This ca --help Show this message and exit. -You can specify either a password via command-line or pass a key file, and you need to specify a destination path where the decrypted backup will be stored. Following is an example usage of `decrypt-backup`: +You can specify the password in the environment variable `MVT_IOS_BACKUP_PASSWORD`, or via command-line argument, or you can pass a key file. You need to specify a destination path where the decrypted backup will be stored. If a password cannot be found and no key file is specified, MVT will ask for a password. Following is an example usage of `decrypt-backup` sending the password via an environment variable: ```bash -mvt-ios decrypt-backup -p password -d /path/to/decrypted /path/to/backup +MVT_IOS_BACKUP_PASSWORD="mypassword" mvt-ios decrypt-backup -d /path/to/decrypted /path/to/backup ``` ## Run `mvt-ios` on a Backup diff --git a/docs/ios/backup/libimobiledevice.md b/docs/ios/backup/libimobiledevice.md index 892717c..5d05c8a 100644 --- a/docs/ios/backup/libimobiledevice.md +++ b/docs/ios/backup/libimobiledevice.md @@ -3,10 +3,14 @@ If you have correctly [installed libimobiledevice](../install.md) you can easily generate an iTunes backup using the `idevicebackup2` tool included in the suite. First, you might want to ensure that backup encryption is enabled (**note: encrypted backup contain more data than unencrypted backups**): ```bash -idevicebackup2 backup encryption on +idevicebackup2 -i backup encryption on ``` -Note that if a backup password was previously set on this device, you might need to use the same or change it. You can try changing password using `idevicebackup2 backup changepw` or resetting the password by resetting only the settings through the iPhone's Settings app. +Note that if a backup password was previously set on this device, you might need to use the same or change it. You can try changing password using `idevicebackup2 -i backup changepw`, or by turning off encryption (`idevicebackup2 -i backup encryption off`) and turning it back on again. + +If you are not able to recover or change the password, you should try to disable encryption and obtain an unencrypted backup. + +If all else fails, as a *last resort* you can try resetting the password by [resetting all the settings through the iPhone's Settings app](https://support.apple.com/en-us/HT205220), via `Settings » General » Reset » Reset All Settings`. Note that resetting the settings through the iPhone's Settings app will wipe some of the files that contain useful forensic traces, so try the options explained above first. Once ready, you can proceed performing the backup: diff --git a/docs/ios/filesystem/dump.md b/docs/ios/filesystem/dump.md index bdaf5a9..7aa537f 100644 --- a/docs/ios/filesystem/dump.md +++ b/docs/ios/filesystem/dump.md @@ -1,6 +1,6 @@ # Dumping the filesystem -While iTunes backup provide a lot of very useful databases and diagnistic data, in some cases you might want to jailbreak the device and perform a full filesystem dump. In that case, you should take a look at [checkra1n](https://checkra.in/), which provides an easy way to obtain root on most recent iPhone models. +While iTunes backup provide a lot of very useful databases and diagnostic data, in some cases you might want to jailbreak the device and perform a full filesystem dump. In that case, you should take a look at [checkra1n](https://checkra.in/), which provides an easy way to obtain root on most recent iPhone models. !!! warning Before you checkra1n any device, make sure you take a full backup, and that you are prepared to do a full factory reset before restoring it. Even after using checkra1n's "Restore System", some traces of the jailbreak are still left on the device and [apps with anti-jailbreaks will be able to detect them](https://github.com/checkra1n/BugTracker/issues/279) and stop functioning. diff --git a/docs/ios/methodology.md b/docs/ios/methodology.md index bbf15d8..637f07e 100644 --- a/docs/ios/methodology.md +++ b/docs/ios/methodology.md @@ -6,7 +6,7 @@ Before jumping into acquiring and analyzing data from an iOS device, you should You will need to decide whether to attempt to jailbreak the device and obtain a full filesystem dump, or not. -While access the full filesystem allows to extact data that would otherwise be unavailable, it might not always be possible to jailbreak a certain iPhone model or version of iOS. In addition, depending on the type of jailbreak available, doing so might compromise some important records, pollute others, or potentially cause unintended malfunctioning of the device later in case it is used again. +While access the full filesystem allows to extract data that would otherwise be unavailable, it might not always be possible to jailbreak a certain iPhone model or version of iOS. In addition, depending on the type of jailbreak available, doing so might compromise some important records, pollute others, or potentially cause unintended malfunctioning of the device later in case it is used again. If you are not expected to return the phone, you might want to consider to attempt a jailbreak after having exhausted all other options, including a backup. diff --git a/docs/ios/records.md b/docs/ios/records.md index f5dad16..1e6f113 100644 --- a/docs/ios/records.md +++ b/docs/ios/records.md @@ -236,7 +236,7 @@ If indicators are provided through the command-line, they are checked against th Backup: :material-close: Full filesystem dump: :material-check: -This JSON file is created by mvt-ios' `WebkitLocalStorage` module. The module extracts a lsit of file and folder names located at the following path */private/var/mobile/Containers/Data/Application/\*/Library/WebKit/WebsiteData/LocalStorage/*, which contains local storage files created by any app installed on the device. +This JSON file is created by mvt-ios' `WebkitLocalStorage` module. The module extracts a list of file and folder names located at the following path */private/var/mobile/Containers/Data/Application/\*/Library/WebKit/WebsiteData/LocalStorage/*, which contains local storage files created by any app installed on the device. If indicators are provided through the command-line, they are checked against the extracted names. Any matches are stored in *webkit_local_storage_detected.json*. diff --git a/mkdocs.yml b/mkdocs.yml index 448b1f6..282b0ef 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -28,6 +28,7 @@ nav: - Welcome: "index.md" - Introduction: "introduction.md" - Installation: "install.md" + - Using Docker: "docker.md" - MVT for iOS: - iOS Forensic Methodology: "ios/methodology.md" - Install libimobiledevice: "ios/install.md" @@ -43,4 +44,5 @@ nav: - Android Forensic Methodology: "android/methodology.md" - Check APKs: "android/download_apks.md" - Check an Android Backup: "android/backup.md" + - Indicators of Compromise: "iocs.md" - License: "license.md" diff --git a/mvt/__init__.py b/mvt/__init__.py index 2512a63..92def7d 100644 --- a/mvt/__init__.py +++ b/mvt/__init__.py @@ -1,4 +1,4 @@ # Mobile Verification Toolkit (MVT) -# Copyright (c) 2021 MVT Project Developers. -# See the file 'LICENSE' for usage and copying permissions, or find a copy at -# https://github.com/mvt-project/mvt/blob/main/LICENSE +# Copyright (c) 2021 The MVT Project Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ diff --git a/mvt/android/__init__.py b/mvt/android/__init__.py index 2ae7e01..249dc30 100644 --- a/mvt/android/__init__.py +++ b/mvt/android/__init__.py @@ -1,6 +1,6 @@ # Mobile Verification Toolkit (MVT) -# Copyright (c) 2021 MVT Project Developers. -# See the file 'LICENSE' for usage and copying permissions, or find a copy at -# https://github.com/mvt-project/mvt/blob/main/LICENSE +# Copyright (c) 2021 The MVT Project Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ from .cli import cli diff --git a/mvt/android/cli.py b/mvt/android/cli.py index 38832e6..945277f 100644 --- a/mvt/android/cli.py +++ b/mvt/android/cli.py @@ -1,17 +1,19 @@ # Mobile Verification Toolkit (MVT) -# Copyright (c) 2021 MVT Project Developers. -# See the file 'LICENSE' for usage and copying permissions, or find a copy at -# https://github.com/mvt-project/mvt/blob/main/LICENSE +# Copyright (c) 2021 The MVT Project Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ -import os -import sys -import click import argparse import logging +import os +import sys + +import click from rich.logging import RichHandler -from mvt.common.module import run_module, save_timeline from mvt.common.indicators import Indicators +from mvt.common.module import run_module, save_timeline + from .download_apks import DownloadAPKs from .lookups.koodous import koodous_lookup from .lookups.virustotal import virustotal_lookup @@ -45,8 +47,8 @@ def cli(): @click.option("--virustotal", "-v", is_flag=True, help="Check packages on VirusTotal") @click.option("--koodous", "-k", is_flag=True, help="Check packages on Koodous") @click.option("--all-checks", "-A", is_flag=True, help="Run all available checks") -@click.option("--output", "-o", type=click.Path(exists=True), - help="Specify a path to a folder where you want to store JSON results") +@click.option("--output", "-o", type=click.Path(exists=False), + help="Specify a path to a folder where you want to store the APKs") @click.option("--from-file", "-f", type=click.Path(exists=True), help="Instead of acquiring from phone, load an existing packages.json file for lookups (mainly for debug purposes)") @click.option("--serial", "-s", type=str, help="Use the Android device with a given serial number") @@ -55,9 +57,12 @@ def download_apks(all_apks, virustotal, koodous, all_checks, output, from_file, if from_file: download = DownloadAPKs.from_json(from_file) else: - if not output: - log.critical("You need to specify an output folder (with --output, -o) when extracting APKs from a device") - sys.exit(-1) + if output and not os.path.exists(output): + try: + os.makedirs(output) + except Exception as e: + log.critical("Unable to create output folder %s: %s", output, e) + sys.exit(-1) download = DownloadAPKs(output_folder=output, all_apks=all_apks, serial=serial) download.run() @@ -82,7 +87,7 @@ def download_apks(all_apks, virustotal, koodous, all_checks, output, from_file, #============================================================================== @cli.command("check-adb", help="Check an Android device over adb") @click.option("--iocs", "-i", type=click.Path(exists=True), help="Path to indicators file") -@click.option("--output", "-o", type=click.Path(exists=True), +@click.option("--output", "-o", type=click.Path(exists=False), help="Specify a path to a folder where you want to store JSON results") @click.option("--list-modules", "-l", is_flag=True, help="Print list of available modules and exit") @click.option("--module", "-m", help="Name of a single module you would like to run instead of all") @@ -97,6 +102,13 @@ def check_adb(iocs, output, list_modules, module, serial): log.info("Checking Android through adb bridge") + if output and not os.path.exists(output): + try: + os.makedirs(output) + except Exception as e: + log.critical("Unable to create output folder %s: %s", output, e) + sys.exit(-1) + if iocs: # Pre-load indicators for performance reasons. log.info("Loading indicators from provided file at %s", iocs) @@ -129,12 +141,19 @@ def check_adb(iocs, output, list_modules, module, serial): #============================================================================== @cli.command("check-backup", help="Check an Android Backup") @click.option("--iocs", "-i", type=click.Path(exists=True), help="Path to indicators file") -@click.option("--output", "-o", type=click.Path(exists=True), help=OUTPUT_HELP_MESSAGE) +@click.option("--output", "-o", type=click.Path(exists=False), help=OUTPUT_HELP_MESSAGE) @click.option("--serial", "-s", type=str, help="Use the Android device with a given serial") @click.argument("BACKUP_PATH", type=click.Path(exists=True)) def check_backup(iocs, output, backup_path, serial): log.info("Checking ADB backup located at: %s", backup_path) + if output and not os.path.exists(output): + try: + os.makedirs(output) + except Exception as e: + log.critical("Unable to create output folder %s: %s", output, e) + sys.exit(-1) + if iocs: # Pre-load indicators for performance reasons. log.info("Loading indicators from provided file at %s", iocs) diff --git a/mvt/android/download_apks.py b/mvt/android/download_apks.py index 6420c5e..3ac03c2 100644 --- a/mvt/android/download_apks.py +++ b/mvt/android/download_apks.py @@ -1,22 +1,24 @@ # Mobile Verification Toolkit (MVT) -# Copyright (c) 2021 MVT Project Developers. -# See the file 'LICENSE' for usage and copying permissions, or find a copy at -# https://github.com/mvt-project/mvt/blob/main/LICENSE +# Copyright (c) 2021 The MVT Project Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ -import os import json import logging +import os + import pkg_resources from tqdm import tqdm from mvt.common.utils import get_sha256_from_file_path + from .modules.adb.base import AndroidExtraction log = logging.getLogger(__name__) # TODO: Would be better to replace tqdm with rich.progress to reduce # the number of dependencies. Need to investigate whether -# it's possible to have a simialr callback system. +# it's possible to have a similar callback system. class PullProgress(tqdm): """PullProgress is a tqdm update system for APK downloads.""" @@ -42,7 +44,7 @@ class DownloadAPKs(AndroidExtraction): """Initialize module. :param output_folder: Path to the folder where data should be stored :param all_apks: Boolean indicating whether to download all packages - or filter known-goods + or filter known-goods :param packages: Provided list of packages, typically for JSON checks :param serial: The USB device serial ID """ diff --git a/mvt/android/lookups/__init__.py b/mvt/android/lookups/__init__.py index 2512a63..92def7d 100644 --- a/mvt/android/lookups/__init__.py +++ b/mvt/android/lookups/__init__.py @@ -1,4 +1,4 @@ # Mobile Verification Toolkit (MVT) -# Copyright (c) 2021 MVT Project Developers. -# See the file 'LICENSE' for usage and copying permissions, or find a copy at -# https://github.com/mvt-project/mvt/blob/main/LICENSE +# Copyright (c) 2021 The MVT Project Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ diff --git a/mvt/android/lookups/koodous.py b/mvt/android/lookups/koodous.py index 78f5c0f..5498bfe 100644 --- a/mvt/android/lookups/koodous.py +++ b/mvt/android/lookups/koodous.py @@ -1,15 +1,15 @@ # Mobile Verification Toolkit (MVT) -# Copyright (c) 2021 MVT Project Developers. -# See the file 'LICENSE' for usage and copying permissions, or find a copy at -# https://github.com/mvt-project/mvt/blob/main/LICENSE +# Copyright (c) 2021 The MVT Project Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ -import requests import logging -from rich.text import Text -from rich.table import Table -from rich.progress import track +import requests from rich.console import Console +from rich.progress import track +from rich.table import Table +from rich.text import Text log = logging.getLogger(__name__) diff --git a/mvt/android/lookups/virustotal.py b/mvt/android/lookups/virustotal.py index eaac727..4fb949f 100644 --- a/mvt/android/lookups/virustotal.py +++ b/mvt/android/lookups/virustotal.py @@ -1,14 +1,15 @@ # Mobile Verification Toolkit (MVT) -# Copyright (c) 2021 MVT Project Developers. -# See the file 'LICENSE' for usage and copying permissions, or find a copy at -# https://github.com/mvt-project/mvt/blob/main/LICENSE +# Copyright (c) 2021 The MVT Project Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +import logging import requests -import logging -from rich.text import Text -from rich.table import Table -from rich.progress import track from rich.console import Console +from rich.progress import track +from rich.table import Table +from rich.text import Text log = logging.getLogger(__name__) diff --git a/mvt/android/modules/__init__.py b/mvt/android/modules/__init__.py index 2512a63..92def7d 100644 --- a/mvt/android/modules/__init__.py +++ b/mvt/android/modules/__init__.py @@ -1,4 +1,4 @@ # Mobile Verification Toolkit (MVT) -# Copyright (c) 2021 MVT Project Developers. -# See the file 'LICENSE' for usage and copying permissions, or find a copy at -# https://github.com/mvt-project/mvt/blob/main/LICENSE +# Copyright (c) 2021 The MVT Project Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ diff --git a/mvt/android/modules/adb/__init__.py b/mvt/android/modules/adb/__init__.py index ddca6b4..e64fbc3 100644 --- a/mvt/android/modules/adb/__init__.py +++ b/mvt/android/modules/adb/__init__.py @@ -1,17 +1,17 @@ # Mobile Verification Toolkit (MVT) -# Copyright (c) 2021 MVT Project Developers. -# See the file 'LICENSE' for usage and copying permissions, or find a copy at -# https://github.com/mvt-project/mvt/blob/main/LICENSE +# Copyright (c) 2021 The MVT Project Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ from .chrome_history import ChromeHistory from .dumpsys_batterystats import DumpsysBatterystats from .dumpsys_packages import DumpsysPackages from .dumpsys_procstats import DumpsysProcstats +from .packages import Packages from .processes import Processes +from .rootbinaries import RootBinaries from .sms import SMS from .whatsapp import Whatsapp -from .packages import Packages -from .rootbinaries import RootBinaries ADB_MODULES = [ChromeHistory, SMS, Whatsapp, Processes, DumpsysBatterystats, DumpsysProcstats, diff --git a/mvt/android/modules/adb/base.py b/mvt/android/modules/adb/base.py index ea0abe5..ec327d5 100644 --- a/mvt/android/modules/adb/base.py +++ b/mvt/android/modules/adb/base.py @@ -1,20 +1,22 @@ # Mobile Verification Toolkit (MVT) -# Copyright (c) 2021 MVT Project Developers. -# See the file 'LICENSE' for usage and copying permissions, or find a copy at -# https://github.com/mvt-project/mvt/blob/main/LICENSE +# Copyright (c) 2021 The MVT Project Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ -import os -import sys -import time import logging +import os +import random +import string +import sys import tempfile +import time from adb_shell.adb_device import AdbDeviceUsb, AdbDeviceTcp from adb_shell.auth.keygen import keygen, write_public_keyfile from adb_shell.auth.sign_pythonrsa import PythonRSASigner -from adb_shell.exceptions import DeviceAuthError, AdbCommandFailureException -from usb1 import USBErrorBusy, USBErrorAccess +from adb_shell.exceptions import AdbCommandFailureException, DeviceAuthError +from usb1 import USBErrorAccess, USBErrorBusy -from mvt.common.module import MVTModule +from mvt.common.module import MVTModule, InsufficientPrivileges log = logging.getLogger(__name__) @@ -106,13 +108,13 @@ class AndroidExtraction(MVTModule): """Check if we have a `su` binary on the Android device. :returns: Boolean indicating whether a `su` binary is present or not """ - return bool(self._adb_command("[ ! -f /sbin/su ] || echo 1")) + return bool(self._adb_command("command -v su")) def _adb_root_or_die(self): """Check if we have a `su` binary, otherwise raise an Exception. """ if not self._adb_check_if_root(): - raise Exception("The Android device does not seem to have a `su` binary. Cannot run this module.") + raise InsufficientPrivileges("The Android device does not seem to have a `su` binary. Cannot run this module.") def _adb_command_as_root(self, command): """Execute an adb shell command. @@ -120,8 +122,23 @@ class AndroidExtraction(MVTModule): :returns: Output of command """ return self._adb_command(f"su -c {command}") + + def _adb_check_file_exists(self, file): + """Verify that a file exists. + :param file: Path of the file + :returns: Boolean indicating whether the file exists or not + """ - def _adb_download(self, remote_path, local_path, progress_callback=None): + # TODO: Need to support checking files without root privileges as well. + + # Connect to the device over adb. + self._adb_connect() + # Check if we have root, if not raise an Exception. + self._adb_root_or_die() + + return bool(self._adb_command_as_root(f"[ ! -f {file} ] || echo 1")) + + def _adb_download(self, remote_path, local_path, progress_callback=None, retry_root=True): """Download a file form the device. :param remote_path: Path to download from the device :param local_path: Path to where to locally store the copy of the file @@ -129,6 +146,37 @@ class AndroidExtraction(MVTModule): """ try: self.device.pull(remote_path, local_path, progress_callback) + except AdbCommandFailureException as e: + if retry_root: + self._adb_download_root(remote_path, local_path, progress_callback) + else: + raise Exception(f"Unable to download file {remote_path}: {e}") + + def _adb_download_root(self, remote_path, local_path, progress_callback=None): + try: + # Check if we have root, if not raise an Exception. + self._adb_root_or_die() + + # We generate a random temporary filename. + tmp_filename = "tmp_" + ''.join(random.choices(string.ascii_uppercase + string.ascii_lowercase + string.digits, k=10)) + + # We create a temporary local file. + new_remote_path = f"/sdcard/{tmp_filename}" + + # We copy the file from the data folder to /sdcard/. + cp = self._adb_command_as_root(f"cp {remote_path} {new_remote_path}") + if cp.startswith("cp: ") and "No such file or directory" in cp: + raise Exception(f"Unable to process file {remote_path}: File not found") + elif cp.startswith("cp: ") and "Permission denied" in cp: + raise Exception(f"Unable to process file {remote_path}: Permission denied") + + # We download from /sdcard/ to the local temporary file. + # If it doesn't work now, don't try again (retry_root=False) + self._adb_download(new_remote_path, local_path, retry_root=False) + + # Delete the copy on /sdcard/. + self._adb_command(f"rm -rf {new_remote_path}") + except AdbCommandFailureException as e: raise Exception(f"Unable to download file {remote_path}: {e}") diff --git a/mvt/android/modules/adb/chrome_history.py b/mvt/android/modules/adb/chrome_history.py index b02a86d..a09a27b 100644 --- a/mvt/android/modules/adb/chrome_history.py +++ b/mvt/android/modules/adb/chrome_history.py @@ -1,13 +1,14 @@ # Mobile Verification Toolkit (MVT) -# Copyright (c) 2021 MVT Project Developers. -# See the file 'LICENSE' for usage and copying permissions, or find a copy at -# https://github.com/mvt-project/mvt/blob/main/LICENSE +# Copyright (c) 2021 The MVT Project Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ +import logging import os import sqlite3 -import logging -from mvt.common.utils import convert_chrometime_to_unix, convert_timestamp_to_iso +from mvt.common.utils import (convert_chrometime_to_unix, + convert_timestamp_to_iso) from .base import AndroidExtraction diff --git a/mvt/android/modules/adb/dumpsys_batterystats.py b/mvt/android/modules/adb/dumpsys_batterystats.py index 986161f..8e66180 100644 --- a/mvt/android/modules/adb/dumpsys_batterystats.py +++ b/mvt/android/modules/adb/dumpsys_batterystats.py @@ -1,10 +1,10 @@ # Mobile Verification Toolkit (MVT) -# Copyright (c) 2021 MVT Project Developers. -# See the file 'LICENSE' for usage and copying permissions, or find a copy at -# https://github.com/mvt-project/mvt/blob/main/LICENSE +# Copyright (c) 2021 The MVT Project Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ -import os import logging +import os from .base import AndroidExtraction diff --git a/mvt/android/modules/adb/dumpsys_packages.py b/mvt/android/modules/adb/dumpsys_packages.py index b1b5260..a6cd1fe 100644 --- a/mvt/android/modules/adb/dumpsys_packages.py +++ b/mvt/android/modules/adb/dumpsys_packages.py @@ -1,10 +1,10 @@ # Mobile Verification Toolkit (MVT) -# Copyright (c) 2021 MVT Project Developers. -# See the file 'LICENSE' for usage and copying permissions, or find a copy at -# https://github.com/mvt-project/mvt/blob/main/LICENSE +# Copyright (c) 2021 The MVT Project Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ -import os import logging +import os from .base import AndroidExtraction diff --git a/mvt/android/modules/adb/dumpsys_procstats.py b/mvt/android/modules/adb/dumpsys_procstats.py index 7fece74..2a5c203 100644 --- a/mvt/android/modules/adb/dumpsys_procstats.py +++ b/mvt/android/modules/adb/dumpsys_procstats.py @@ -1,10 +1,10 @@ # Mobile Verification Toolkit (MVT) -# Copyright (c) 2021 MVT Project Developers. -# See the file 'LICENSE' for usage and copying permissions, or find a copy at -# https://github.com/mvt-project/mvt/blob/main/LICENSE +# Copyright (c) 2021 The MVT Project Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ -import os import logging +import os from .base import AndroidExtraction diff --git a/mvt/android/modules/adb/packages.py b/mvt/android/modules/adb/packages.py index fa87de9..09cc7a0 100644 --- a/mvt/android/modules/adb/packages.py +++ b/mvt/android/modules/adb/packages.py @@ -1,10 +1,11 @@ # Mobile Verification Toolkit (MVT) -# Copyright (c) 2021 MVT Project Developers. -# See the file 'LICENSE' for usage and copying permissions, or find a copy at -# https://github.com/mvt-project/mvt/blob/main/LICENSE +# Copyright (c) 2021 The MVT Project Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ -import os import logging +import os + import pkg_resources from .base import AndroidExtraction diff --git a/mvt/android/modules/adb/processes.py b/mvt/android/modules/adb/processes.py index 0052abf..ad00cf7 100644 --- a/mvt/android/modules/adb/processes.py +++ b/mvt/android/modules/adb/processes.py @@ -1,7 +1,7 @@ # Mobile Verification Toolkit (MVT) -# Copyright (c) 2021 MVT Project Developers. -# See the file 'LICENSE' for usage and copying permissions, or find a copy at -# https://github.com/mvt-project/mvt/blob/main/LICENSE +# Copyright (c) 2021 The MVT Project Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ import logging diff --git a/mvt/android/modules/adb/rootbinaries.py b/mvt/android/modules/adb/rootbinaries.py index 1804e81..c6ff128 100644 --- a/mvt/android/modules/adb/rootbinaries.py +++ b/mvt/android/modules/adb/rootbinaries.py @@ -1,10 +1,11 @@ # Mobile Verification Toolkit (MVT) -# Copyright (c) 2021 MVT Project Developers. -# See the file 'LICENSE' for usage and copying permissions, or find a copy at -# https://github.com/mvt-project/mvt/blob/main/LICENSE +# Copyright (c) 2021 The MVT Project Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ -import os import logging +import os + import pkg_resources from .base import AndroidExtraction diff --git a/mvt/android/modules/adb/sms.py b/mvt/android/modules/adb/sms.py index 9b0c431..dc9be51 100644 --- a/mvt/android/modules/adb/sms.py +++ b/mvt/android/modules/adb/sms.py @@ -1,18 +1,43 @@ # Mobile Verification Toolkit (MVT) -# Copyright (c) 2021 MVT Project Developers. -# See the file 'LICENSE' for usage and copying permissions, or find a copy at -# https://github.com/mvt-project/mvt/blob/main/LICENSE +# Copyright (c) 2021 The MVT Project Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ +import logging import os import sqlite3 -import logging + +from mvt.common.utils import check_for_links, convert_timestamp_to_iso from .base import AndroidExtraction -from mvt.common.utils import convert_timestamp_to_iso, check_for_links log = logging.getLogger(__name__) -SMS_PATH = "data/data/com.google.android.apps.messaging/databases/bugle_db" +SMS_BUGLE_PATH = "data/data/com.google.android.apps.messaging/databases/bugle_db" +SMS_BUGLE_QUERY = """ +SELECT + ppl.normalized_destination AS number, + p.timestamp AS timestamp, +CASE WHEN m.sender_id IN +(SELECT _id FROM participants WHERE contact_id=-1) +THEN 2 ELSE 1 END incoming, p.text AS text +FROM messages m, conversations c, parts p, + participants ppl, conversation_participants cp +WHERE (m.conversation_id = c._id) + AND (m._id = p.message_id) + AND (cp.conversation_id = c._id) + AND (cp.participant_id = ppl._id); +""" + +SMS_MMSSMS_PATH = "data/data/com.android.providers.telephony/databases/mmssms.db" +SMS_MMSMS_QUERY = """ +SELECT + address AS number, + date_sent AS timestamp, + type as incoming, + body AS text +FROM sms; +""" class SMS(AndroidExtraction): """This module extracts all SMS messages containing links.""" @@ -50,20 +75,12 @@ class SMS(AndroidExtraction): """ conn = sqlite3.connect(db_path) cur = conn.cursor() - cur.execute(""" - SELECT - ppl.normalized_destination AS number, - p.timestamp AS timestamp, - CASE WHEN m.sender_id IN - (SELECT _id FROM participants WHERE contact_id=-1) - THEN 2 ELSE 1 END incoming, p.text AS text - FROM messages m, conversations c, parts p, - participants ppl, conversation_participants cp - WHERE (m.conversation_id = c._id) - AND (m._id = p.message_id) - AND (cp.conversation_id = c._id) - AND (cp.participant_id = ppl._id); - """) + + if (self.SMS_DB_TYPE == 1): + cur.execute(SMS_BUGLE_QUERY) + elif (self.SMS_DB_TYPE == 2): + cur.execute(SMS_MMSMS_QUERY) + names = [description[0] for description in cur.description] for item in cur: @@ -85,7 +102,15 @@ class SMS(AndroidExtraction): log.info("Extracted a total of %d SMS messages containing links", len(self.results)) def run(self): + # Checking the SMS database path try: - self._adb_process_file(os.path.join("/", SMS_PATH), self._parse_db) + if (self._adb_check_file_exists(os.path.join("/", SMS_BUGLE_PATH))): + self.SMS_DB_TYPE = 1 + self._adb_process_file(os.path.join("/", SMS_BUGLE_PATH), self._parse_db) + elif (self._adb_check_file_exists(os.path.join("/", SMS_MMSSMS_PATH))): + self.SMS_DB_TYPE = 2 + self._adb_process_file(os.path.join("/", SMS_MMSSMS_PATH), self._parse_db) + else: + self.log.error("No SMS database found") except Exception as e: self.log.error(e) diff --git a/mvt/android/modules/adb/whatsapp.py b/mvt/android/modules/adb/whatsapp.py index 68db6c2..11482c5 100644 --- a/mvt/android/modules/adb/whatsapp.py +++ b/mvt/android/modules/adb/whatsapp.py @@ -1,14 +1,16 @@ # Mobile Verification Toolkit (MVT) -# Copyright (c) 2021 MVT Project Developers. -# See the file 'LICENSE' for usage and copying permissions, or find a copy at -# https://github.com/mvt-project/mvt/blob/main/LICENSE +# Copyright (c) 2021 The MVT Project Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ +import base64 +import logging import os import sqlite3 -import logging + +from mvt.common.utils import check_for_links, convert_timestamp_to_iso from .base import AndroidExtraction -from mvt.common.utils import convert_timestamp_to_iso, check_for_links log = logging.getLogger(__name__) @@ -69,6 +71,8 @@ class Whatsapp(AndroidExtraction): # If we find links in the messages or if they are empty we add them to the list. if check_for_links(message["data"]) or message["data"].strip() == "": + if (message.get('thumb_image') is not None): + message['thumb_image'] = base64.b64encode(message['thumb_image']) messages.append(message) cur.close() diff --git a/mvt/android/modules/backup/__init__.py b/mvt/android/modules/backup/__init__.py index 138b300..10736b7 100644 --- a/mvt/android/modules/backup/__init__.py +++ b/mvt/android/modules/backup/__init__.py @@ -1,7 +1,7 @@ # Mobile Verification Toolkit (MVT) -# Copyright (c) 2021 MVT Project Developers. -# See the file 'LICENSE' for usage and copying permissions, or find a copy at -# https://github.com/mvt-project/mvt/blob/main/LICENSE +# Copyright (c) 2021 The MVT Project Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ from .sms import SMS diff --git a/mvt/android/modules/backup/sms.py b/mvt/android/modules/backup/sms.py index 86cc54b..b6919f2 100644 --- a/mvt/android/modules/backup/sms.py +++ b/mvt/android/modules/backup/sms.py @@ -1,15 +1,15 @@ # Mobile Verification Toolkit (MVT) -# Copyright (c) 2021 MVT Project Developers. -# See the file 'LICENSE' for usage and copying permissions, or find a copy at -# https://github.com/mvt-project/mvt/blob/main/LICENSE +# Copyright (c) 2021 The MVT Project Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ -import os import json +import os import zlib from mvt.common.module import MVTModule -from mvt.common.utils import check_for_links -from mvt.common.utils import convert_timestamp_to_iso +from mvt.common.utils import check_for_links, convert_timestamp_to_iso + class SMS(MVTModule): diff --git a/mvt/common/__init__.py b/mvt/common/__init__.py index 2512a63..92def7d 100644 --- a/mvt/common/__init__.py +++ b/mvt/common/__init__.py @@ -1,4 +1,4 @@ # Mobile Verification Toolkit (MVT) -# Copyright (c) 2021 MVT Project Developers. -# See the file 'LICENSE' for usage and copying permissions, or find a copy at -# https://github.com/mvt-project/mvt/blob/main/LICENSE +# Copyright (c) 2021 The MVT Project Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ diff --git a/mvt/common/indicators.py b/mvt/common/indicators.py index 9532295..a53dc89 100644 --- a/mvt/common/indicators.py +++ b/mvt/common/indicators.py @@ -1,13 +1,14 @@ # Mobile Verification Toolkit (MVT) -# Copyright (c) 2021 MVT Project Developers. -# See the file 'LICENSE' for usage and copying permissions, or find a copy at -# https://github.com/mvt-project/mvt/blob/main/LICENSE +# Copyright (c) 2021 The MVT Project Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ -import os import json +import os from .url import URL + class Indicators: """This class is used to parse indicators from a STIX2 file and provide functions to compare extracted artifacts to the indicators. diff --git a/mvt/common/module.py b/mvt/common/module.py index 7e287e3..e33b416 100644 --- a/mvt/common/module.py +++ b/mvt/common/module.py @@ -1,18 +1,29 @@ # Mobile Verification Toolkit (MVT) -# Copyright (c) 2021 MVT Project Developers. -# See the file 'LICENSE' for usage and copying permissions, or find a copy at -# https://github.com/mvt-project/mvt/blob/main/LICENSE +# Copyright (c) 2021 The MVT Project Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ -import io -import os -import re import csv import glob +import io import logging +import os +import re + import simplejson as json from .indicators import Indicators + +class DatabaseNotFoundError(Exception): + pass + +class DatabaseCorruptedError(Exception): + pass + +class InsufficientPrivileges(Exception): + pass + class MVTModule(object): """This class provides a base for all extraction modules.""" @@ -69,7 +80,7 @@ class MVTModule(object): self.indicators = Indicators(file_path, self.log) def check_indicators(self): - """Check the results of this module againt a provided list of + """Check the results of this module against a provided list of indicators.""" raise NotImplementedError @@ -85,7 +96,11 @@ class MVTModule(object): results_file_name = f"{name}.json" results_json_path = os.path.join(self.output_folder, results_file_name) with open(results_json_path, "w") as handle: - json.dump(self.results, handle, indent=4) + try: + json.dump(self.results, handle, indent=4) + except Exception as e: + self.log.error("Unable to store results of module %s to file %s: %s", + self.__class__.__name__, results_file_name, e) if self.detected: detected_file_name = f"{name}_detected.json" @@ -101,19 +116,21 @@ class MVTModule(object): """ for result in self.results: record = self.serialize(result) - if type(record) == list: - self.timeline.extend(record) - else: - self.timeline.append(record) + if record: + if type(record) == list: + self.timeline.extend(record) + else: + self.timeline.append(record) for detected in self.detected: record = self.serialize(detected) - if type(record) == list: - self.timeline_detected.extend(record) - else: - self.timeline_detected.append(record) + if record: + if type(record) == list: + self.timeline_detected.extend(record) + else: + self.timeline_detected.append(record) - # De-duplicate timeline entries + # De-duplicate timeline entries. self.timeline = self.timeline_deduplicate(self.timeline) self.timeline_detected = self.timeline_deduplicate(self.timeline_detected) @@ -138,8 +155,13 @@ def run_module(module): except NotImplementedError: module.log.exception("The run() procedure of module %s was not implemented yet!", module.__class__.__name__) - except FileNotFoundError as e: - module.log.error("There might be no data to extract by module %s: %s", + except InsufficientPrivileges as e: + module.log.info("Insufficient privileges for module %s: %s", module.__class__.__name__, e) + except DatabaseNotFoundError as e: + module.log.info("There might be no data to extract by module %s: %s", + module.__class__.__name__, e) + except DatabaseCorruptedError as e: + module.log.error("The %s module database seems to be corrupted and recovery failed: %s", module.__class__.__name__, e) except Exception as e: module.log.exception("Error in running extraction from module %s: %s", diff --git a/mvt/common/options.py b/mvt/common/options.py index bc0842b..10d724c 100644 --- a/mvt/common/options.py +++ b/mvt/common/options.py @@ -1,11 +1,12 @@ # Mobile Verification Toolkit (MVT) -# Copyright (c) 2021 MVT Project Developers. -# See the file 'LICENSE' for usage and copying permissions, or find a copy at -# https://github.com/mvt-project/mvt/blob/main/LICENSE +# Copyright (c) 2021 The MVT Project Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ # From: https://gist.github.com/stanchan/bce1c2d030c76fe9223b5ff6ad0f03db -from click import command, option, Option, UsageError +from click import Option, UsageError, command, option + class MutuallyExclusiveOption(Option): """This class extends click to support mutually exclusive options. diff --git a/mvt/common/url.py b/mvt/common/url.py index 57c7723..df04e55 100644 --- a/mvt/common/url.py +++ b/mvt/common/url.py @@ -1,7 +1,7 @@ # Mobile Verification Toolkit (MVT) -# Copyright (c) 2021 MVT Project Developers. -# See the file 'LICENSE' for usage and copying permissions, or find a copy at -# https://github.com/mvt-project/mvt/blob/main/LICENSE +# Copyright (c) 2021 The MVT Project Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ import requests from tld import get_tld diff --git a/mvt/common/utils.py b/mvt/common/utils.py index 6805704..25d6cb1 100644 --- a/mvt/common/utils.py +++ b/mvt/common/utils.py @@ -1,12 +1,13 @@ # Mobile Verification Toolkit (MVT) -# Copyright (c) 2021 MVT Project Developers. -# See the file 'LICENSE' for usage and copying permissions, or find a copy at -# https://github.com/mvt-project/mvt/blob/main/LICENSE +# Copyright (c) 2021 The MVT Project Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ -import os -import re import datetime import hashlib +import os +import re + def convert_mactime_to_unix(timestamp, from_2001=True): """Converts Mac Standard Time to a Unix timestamp. diff --git a/mvt/ios/__init__.py b/mvt/ios/__init__.py index 2ae7e01..249dc30 100644 --- a/mvt/ios/__init__.py +++ b/mvt/ios/__init__.py @@ -1,6 +1,6 @@ # Mobile Verification Toolkit (MVT) -# Copyright (c) 2021 MVT Project Developers. -# See the file 'LICENSE' for usage and copying permissions, or find a copy at -# https://github.com/mvt-project/mvt/blob/main/LICENSE +# Copyright (c) 2021 The MVT Project Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ from .cli import cli diff --git a/mvt/ios/cli.py b/mvt/ios/cli.py index 47a45cb..c98d9f7 100644 --- a/mvt/ios/cli.py +++ b/mvt/ios/cli.py @@ -1,18 +1,20 @@ # Mobile Verification Toolkit (MVT) -# Copyright (c) 2021 MVT Project Developers. -# See the file 'LICENSE' for usage and copying permissions, or find a copy at -# https://github.com/mvt-project/mvt/blob/main/LICENSE +# Copyright (c) 2021 The MVT Project Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ +import logging import os import sys -import click import tarfile -import logging -from rich.logging import RichHandler +import click +from rich.logging import RichHandler +from rich.prompt import Prompt + +from mvt.common.indicators import Indicators from mvt.common.module import run_module, save_timeline from mvt.common.options import MutuallyExclusiveOption -from mvt.common.indicators import Indicators from .decrypt import DecryptBackup from .modules.fs import BACKUP_MODULES, FS_MODULES @@ -26,6 +28,8 @@ log = logging.getLogger(__name__) # Help messages of repeating options. OUTPUT_HELP_MESSAGE = "Specify a path to a folder where you want to store JSON results" +# Set this environment variable to a password if needed. +PASSWD_ENV = "MVT_IOS_BACKUP_PASSWORD" #============================================================================== # Main @@ -42,8 +46,7 @@ def cli(): @click.option("--destination", "-d", required=True, help="Path to the folder where to store the decrypted backup") @click.option("--password", "-p", cls=MutuallyExclusiveOption, - help="Password to use to decrypt the backup", - prompt="Enter backup password", hide_input=True, prompt_required=False, + help=f"Password to use to decrypt the backup (or, set {PASSWD_ENV} environment variable)", mutually_exclusive=["key_file"]) @click.option("--key-file", "-k", cls=MutuallyExclusiveOption, type=click.Path(exists=True), @@ -52,13 +55,59 @@ def cli(): @click.argument("BACKUP_PATH", type=click.Path(exists=True)) def decrypt_backup(destination, password, key_file, backup_path): backup = DecryptBackup(backup_path, destination) - if password: - backup.decrypt_with_password(password) - elif key_file: + + if key_file: + if PASSWD_ENV in os.environ: + log.info(f"Ignoring {PASSWD_ENV} environment variable, using --key-file '{key_file}' instead") + backup.decrypt_with_key_file(key_file) + elif password: + log.info("Your password may be visible in the process table because it was supplied on the command line!") + + if PASSWD_ENV in os.environ: + log.info(f"Ignoring {PASSWD_ENV} environment variable, using --password argument instead") + + backup.decrypt_with_password(password) + elif PASSWD_ENV in os.environ: + log.info(f"Using password from {PASSWD_ENV} environment variable") + backup.decrypt_with_password(os.environ[PASSWD_ENV]) else: - raise click.ClickException("Missing required option. Specify either " - "--password or --key-file.") + sekrit = Prompt.ask("Enter backup password", password=True) + backup.decrypt_with_password(sekrit) + + backup.process_backup() + + +#============================================================================== +# Command: extract-key +#============================================================================== +@cli.command("extract-key", help="Extract decryption key from an iTunes backup") +@click.option("--password", "-p", + help=f"Password to use to decrypt the backup (or, set {PASSWD_ENV} environment variable)") +@click.option("--key-file", "-k", + help="Key file to be written (if unset, will print to STDOUT)", + required=False, + type=click.Path(exists=False, file_okay=True, dir_okay=False, writable=True)) +@click.argument("BACKUP_PATH", type=click.Path(exists=True)) +def extract_key(password, backup_path, key_file): + backup = DecryptBackup(backup_path) + + if password: + log.info("Your password may be visible in the process table because it was supplied on the command line!") + + if PASSWD_ENV in os.environ: + log.info(f"Ignoring {PASSWD_ENV} environment variable, using --password argument instead") + elif PASSWD_ENV in os.environ: + log.info(f"Using password from {PASSWD_ENV} environment variable") + password = os.environ[PASSWD_ENV] + else: + password = Prompt.ask("Enter backup password", password=True) + + backup.decrypt_with_password(password) + backup.get_key() + + if key_file: + backup.write_key(key_file) #============================================================================== @@ -66,7 +115,7 @@ def decrypt_backup(destination, password, key_file, backup_path): #============================================================================== @cli.command("check-backup", help="Extract artifacts from an iTunes backup") @click.option("--iocs", "-i", type=click.Path(exists=True), help="Path to indicators file") -@click.option("--output", "-o", type=click.Path(exists=True), help=OUTPUT_HELP_MESSAGE) +@click.option("--output", "-o", type=click.Path(exists=False), help=OUTPUT_HELP_MESSAGE) @click.option("--fast", "-f", is_flag=True, help="Avoid running time/resource consuming features") @click.option("--list-modules", "-l", is_flag=True, help="Print list of available modules and exit") @click.option("--module", "-m", help="Name of a single module you would like to run instead of all") @@ -81,6 +130,13 @@ def check_backup(iocs, output, fast, backup_path, list_modules, module): log.info("Checking iTunes backup located at: %s", backup_path) + if output and not os.path.exists(output): + try: + os.makedirs(output) + except Exception as e: + log.critical("Unable to create output folder %s: %s", output, e) + sys.exit(-1) + if iocs: # Pre-load indicators for performance reasons. log.info("Loading indicators from provided file at: %s", iocs) @@ -116,7 +172,7 @@ def check_backup(iocs, output, fast, backup_path, list_modules, module): #============================================================================== @cli.command("check-fs", help="Extract artifacts from a full filesystem dump") @click.option("--iocs", "-i", type=click.Path(exists=True), help="Path to indicators file") -@click.option("--output", "-o", type=click.Path(exists=True), help=OUTPUT_HELP_MESSAGE) +@click.option("--output", "-o", type=click.Path(exists=False), help=OUTPUT_HELP_MESSAGE) @click.option("--fast", "-f", is_flag=True, help="Avoid running time/resource consuming features") @click.option("--list-modules", "-l", is_flag=True, help="Print list of available modules and exit") @click.option("--module", "-m", help="Name of a single module you would like to run instead of all") @@ -131,6 +187,13 @@ def check_fs(iocs, output, fast, dump_path, list_modules, module): log.info("Checking filesystem dump located at: %s", dump_path) + if output and not os.path.exists(output): + try: + os.makedirs(output) + except Exception as e: + log.critical("Unable to create output folder %s: %s", output, e) + sys.exit(-1) + if iocs: # Pre-load indicators for performance reasons. log.info("Loading indicators from provided file at: %s", iocs) diff --git a/mvt/ios/decrypt.py b/mvt/ios/decrypt.py index 7b1acb0..fe50af5 100644 --- a/mvt/ios/decrypt.py +++ b/mvt/ios/decrypt.py @@ -1,13 +1,14 @@ # Mobile Verification Toolkit (MVT) -# Copyright (c) 2021 MVT Project Developers. -# See the file 'LICENSE' for usage and copying permissions, or find a copy at -# https://github.com/mvt-project/mvt/blob/main/LICENSE +# Copyright (c) 2021 The MVT Project Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ +import binascii +import logging import os import shutil import sqlite3 -import logging -import binascii + from iOSbackup import iOSbackup log = logging.getLogger(__name__) @@ -17,7 +18,7 @@ class DecryptBackup: using either a password or a key file. """ - def __init__(self, backup_path, dest_path): + def __init__(self, backup_path, dest_path=None): """Decrypts an encrypted iOS backup. :param backup_path: Path to the encrypted backup folder :param dest_path: Path to the folder where to store the decrypted backup @@ -25,8 +26,12 @@ class DecryptBackup: self.backup_path = backup_path self.dest_path = dest_path self._backup = None + self._decryption_key = None + + def process_backup(self): + if not os.path.exists(self.dest_path): + os.makedirs(self.dest_path) - def _process_backup(self): manifest_path = os.path.join(self.dest_path, "Manifest.db") # We extract a decrypted Manifest.db. self._backup.getManifestDB() @@ -68,19 +73,18 @@ class DecryptBackup: """ log.info("Decrypting iOS backup at path %s with password", self.backup_path) - if not os.path.exists(self.dest_path): - os.makedirs(self.dest_path) - try: self._backup = iOSbackup(udid=os.path.basename(self.backup_path), cleartextpassword=password, backuproot=os.path.dirname(self.backup_path)) except Exception as e: - log.exception(e) - log.critical("Failed to decrypt backup. Did you provide the correct password?") - return - else: - self._process_backup() + if isinstance(e, KeyError) and len(e.args) > 0 and e.args[0] == b"KEY": + log.critical("Failed to decrypt backup. Password is probably wrong.") + elif isinstance(e, FileNotFoundError) and os.path.basename(e.filename) == "Manifest.plist": + log.critical(f"Failed to find a valid backup at {self.backup_path}. Did you point to the right backup path?") + else: + log.exception(e) + log.critical("Failed to decrypt backup. Did you provide the correct password? Did you point to the right backup path?") def decrypt_with_key_file(self, key_file): """Decrypts an encrypted iOS backup using a key file. @@ -89,9 +93,6 @@ class DecryptBackup: log.info("Decrypting iOS backup at path %s with key file %s", self.backup_path, key_file) - if not os.path.exists(self.dest_path): - os.makedirs(self.dest_path) - with open(key_file, "rb") as handle: key_bytes = handle.read() @@ -108,6 +109,31 @@ class DecryptBackup: except Exception as e: log.exception(e) log.critical("Failed to decrypt backup. Did you provide the correct key file?") + + def get_key(self): + """Retrieve and prints the encryption key. + """ + if not self._backup: + return + + self._decryption_key = self._backup.getDecryptionKey() + log.info("Derived decryption key for backup at path %s is: \"%s\"", + self.backup_path, self._decryption_key) + + def write_key(self, key_path): + """Save extracted key to file. + :param key_path: Path to the file where to write the derived decryption key. + """ + if not self._decryption_key: + return + + try: + with open(key_path, 'w') as handle: + handle.write(self._decryption_key) + except Exception as e: + log.exception(e) + log.critical("Failed to write key to file: %s", key_path) return else: - self._process_backup() + log.info("Wrote decryption key to file: %s. This file is equivalent to a plaintext password. Keep it safe!", + key_path) diff --git a/mvt/ios/modules/__init__.py b/mvt/ios/modules/__init__.py index 2512a63..92def7d 100644 --- a/mvt/ios/modules/__init__.py +++ b/mvt/ios/modules/__init__.py @@ -1,4 +1,4 @@ # Mobile Verification Toolkit (MVT) -# Copyright (c) 2021 MVT Project Developers. -# See the file 'LICENSE' for usage and copying permissions, or find a copy at -# https://github.com/mvt-project/mvt/blob/main/LICENSE +# Copyright (c) 2021 The MVT Project Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ diff --git a/mvt/ios/modules/fs/__init__.py b/mvt/ios/modules/fs/__init__.py index dad5124..7fec68e 100644 --- a/mvt/ios/modules/fs/__init__.py +++ b/mvt/ios/modules/fs/__init__.py @@ -1,42 +1,44 @@ # Mobile Verification Toolkit (MVT) -# Copyright (c) 2021 MVT Project Developers. -# See the file 'LICENSE' for usage and copying permissions, or find a copy at -# https://github.com/mvt-project/mvt/blob/main/LICENSE +# Copyright (c) 2021 The MVT Project Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ -from .manifest import Manifest -from .contacts import Contacts -from .net_netusage import Netusage -from .net_datausage import Datausage -from .safari_history import SafariHistory -from .safari_favicon import SafariFavicon -from .safari_browserstate import SafariBrowserState -from .webkit_indexeddb import WebkitIndexedDB -from .webkit_localstorage import WebkitLocalStorage -from .webkit_safariviewservice import WebkitSafariViewService -from .webkit_session_resource_log import WebkitSessionResourceLog -from .chrome_history import ChromeHistory +from .cache_files import CacheFiles +from .calls import Calls from .chrome_favicon import ChromeFavicon -from .firefox_history import FirefoxHistory +from .chrome_history import ChromeHistory +from .contacts import Contacts +from .filesystem import Filesystem from .firefox_favicon import FirefoxFavicon -from .version_history import IOSVersionHistory +from .firefox_history import FirefoxHistory from .idstatuscache import IDStatusCache -from .locationd import LocationdClients from .interactionc import InteractionC +from .locationd import LocationdClients +from .manifest import Manifest +from .net_datausage import Datausage +from .net_netusage import Netusage +from .safari_browserstate import SafariBrowserState +from .safari_favicon import SafariFavicon +from .safari_history import SafariHistory from .sms import SMS from .sms_attachments import SMSAttachments -from .calls import Calls +from .version_history import IOSVersionHistory +from .webkit_indexeddb import WebkitIndexedDB +from .webkit_localstorage import WebkitLocalStorage +from .webkit_resource_load_statistics import WebkitResourceLoadStatistics +from .webkit_safariviewservice import WebkitSafariViewService +from .webkit_session_resource_log import WebkitSessionResourceLog from .whatsapp import Whatsapp -from .cache_files import CacheFiles -from .filesystem import Filesystem BACKUP_MODULES = [SafariBrowserState, SafariHistory, Datausage, SMS, SMSAttachments, ChromeHistory, ChromeFavicon, WebkitSessionResourceLog, - Calls, IDStatusCache, LocationdClients, InteractionC, - FirefoxHistory, FirefoxFavicon, Contacts, Manifest, Whatsapp] + WebkitResourceLoadStatistics, Calls, IDStatusCache, LocationdClients, + InteractionC, FirefoxHistory, FirefoxFavicon, Contacts, Manifest, Whatsapp] FS_MODULES = [IOSVersionHistory, SafariHistory, SafariFavicon, SafariBrowserState, WebkitIndexedDB, WebkitLocalStorage, WebkitSafariViewService, - WebkitSessionResourceLog, Datausage, Netusage, ChromeHistory, + WebkitResourceLoadStatistics, WebkitSessionResourceLog, + Datausage, Netusage, ChromeHistory, ChromeFavicon, Calls, IDStatusCache, SMS, SMSAttachments, LocationdClients, InteractionC, FirefoxHistory, FirefoxFavicon, Contacts, CacheFiles, Whatsapp, Filesystem] diff --git a/mvt/ios/modules/fs/base.py b/mvt/ios/modules/fs/base.py index 7a5437d..2306be2 100644 --- a/mvt/ios/modules/fs/base.py +++ b/mvt/ios/modules/fs/base.py @@ -1,12 +1,18 @@ # Mobile Verification Toolkit (MVT) -# Copyright (c) 2021 MVT Project Developers. -# See the file 'LICENSE' for usage and copying permissions, or find a copy at -# https://github.com/mvt-project/mvt/blob/main/LICENSE +# Copyright (c) 2021 The MVT Project Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ -import os import glob +import io +import os +import shutil +import sqlite3 +import subprocess + +from mvt.common.module import (DatabaseCorruptedError, DatabaseNotFoundError, + MVTModule) -from mvt.common.module import MVTModule class IOSExtraction(MVTModule): """This class provides a base for all iOS filesystem/backup extraction modules.""" @@ -15,10 +21,53 @@ class IOSExtraction(MVTModule): is_fs_dump = False is_sysdiagnose = False + def _is_database_malformed(self, file_path): + # Check if the database is malformed. + conn = sqlite3.connect(file_path) + cur = conn.cursor() + + try: + recover = False + cur.execute("SELECT name FROM sqlite_master WHERE type='table';") + except sqlite3.DatabaseError as e: + if "database disk image is malformed" in str(e): + recover = True + finally: + conn.close() + + return recover + + def _recover_database(self, file_path): + """Tries to recover a malformed database by running a .clone command. + :param file_path: Path to the malformed database file. + """ + # TODO: Find a better solution. + + self.log.info("Database at path %s is malformed. Trying to recover...", file_path) + + if not os.path.exists(file_path): + return + + if not shutil.which("sqlite3"): + raise DatabaseCorruptedError("Unable to recover without sqlite3 binary. Please install sqlite3!") + if '"' in file_path: + raise DatabaseCorruptedError(f"Database at path '{file_path}' is corrupted. unable to recover because it has a quotation mark (\") in its name.") + + bak_path = f"{file_path}.bak" + shutil.move(file_path, bak_path) + + ret = subprocess.call(["sqlite3", bak_path, f".clone \"{file_path}\""], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if ret != 0: + raise DatabaseCorruptedError("Recovery of database failed") + + self.log.info("Database at path %s recovered successfully!", file_path) + def _find_ios_database(self, backup_ids=None, root_paths=[]): """Try to locate the module's database file from either an iTunes backup or a full filesystem dump. :param backup_id: iTunes backup database file's ID (or hash). + :param root_paths: Glob patterns for files to seek in filesystem dump. """ file_path = None # First we check if the was an explicit file path specified. @@ -52,4 +101,7 @@ class IOSExtraction(MVTModule): if file_path: self.file_path = file_path else: - raise FileNotFoundError("Unable to find the module's database file") + raise DatabaseNotFoundError("Unable to find the module's database file") + + if self._is_database_malformed(self.file_path): + self._recover_database(self.file_path) diff --git a/mvt/ios/modules/fs/cache_files.py b/mvt/ios/modules/fs/cache_files.py index ef35931..80fd166 100644 --- a/mvt/ios/modules/fs/cache_files.py +++ b/mvt/ios/modules/fs/cache_files.py @@ -1,13 +1,14 @@ # Mobile Verification Toolkit (MVT) -# Copyright (c) 2021 MVT Project Developers. -# See the file 'LICENSE' for usage and copying permissions, or find a copy at -# https://github.com/mvt-project/mvt/blob/main/LICENSE +# Copyright (c) 2021 The MVT Project Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ import os import sqlite3 from .base import IOSExtraction + class CacheFiles(IOSExtraction): def __init__(self, file_path=None, base_folder=None, output_folder=None, diff --git a/mvt/ios/modules/fs/calls.py b/mvt/ios/modules/fs/calls.py index 2ed3de5..45c91ea 100644 --- a/mvt/ios/modules/fs/calls.py +++ b/mvt/ios/modules/fs/calls.py @@ -1,11 +1,12 @@ # Mobile Verification Toolkit (MVT) -# Copyright (c) 2021 MVT Project Developers. -# See the file 'LICENSE' for usage and copying permissions, or find a copy at -# https://github.com/mvt-project/mvt/blob/main/LICENSE +# Copyright (c) 2021 The MVT Project Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ import sqlite3 from mvt.common.utils import convert_mactime_to_unix, convert_timestamp_to_iso + from .base import IOSExtraction CALLS_BACKUP_IDS = [ diff --git a/mvt/ios/modules/fs/chrome_favicon.py b/mvt/ios/modules/fs/chrome_favicon.py index 666e7c8..7b6014b 100644 --- a/mvt/ios/modules/fs/chrome_favicon.py +++ b/mvt/ios/modules/fs/chrome_favicon.py @@ -1,11 +1,12 @@ # Mobile Verification Toolkit (MVT) -# Copyright (c) 2021 MVT Project Developers. -# See the file 'LICENSE' for usage and copying permissions, or find a copy at -# https://github.com/mvt-project/mvt/blob/main/LICENSE +# Copyright (c) 2021 The MVT Project Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ import sqlite3 -from mvt.common.utils import convert_chrometime_to_unix, convert_timestamp_to_iso +from mvt.common.utils import (convert_chrometime_to_unix, + convert_timestamp_to_iso) from .base import IOSExtraction diff --git a/mvt/ios/modules/fs/chrome_history.py b/mvt/ios/modules/fs/chrome_history.py index 8d8a64e..e0573a5 100644 --- a/mvt/ios/modules/fs/chrome_history.py +++ b/mvt/ios/modules/fs/chrome_history.py @@ -1,11 +1,12 @@ # Mobile Verification Toolkit (MVT) -# Copyright (c) 2021 MVT Project Developers. -# See the file 'LICENSE' for usage and copying permissions, or find a copy at -# https://github.com/mvt-project/mvt/blob/main/LICENSE +# Copyright (c) 2021 The MVT Project Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ import sqlite3 -from mvt.common.utils import convert_chrometime_to_unix, convert_timestamp_to_iso +from mvt.common.utils import (convert_chrometime_to_unix, + convert_timestamp_to_iso) from .base import IOSExtraction @@ -35,6 +36,14 @@ class ChromeHistory(IOSExtraction): "data": f"{record['id']} - {record['url']} (visit ID: {record['visit_id']}, redirect source: {record['redirect_source']})" } + def check_indicators(self): + if not self.indicators: + return + + for result in self.results: + if self.indicators.check_domain(result["url"]): + self.detected.append(result) + def run(self): self._find_ios_database(backup_ids=CHROME_HISTORY_BACKUP_IDS, root_paths=CHROME_HISTORY_ROOT_PATHS) self.log.info("Found Chrome history database at path: %s", self.file_path) diff --git a/mvt/ios/modules/fs/contacts.py b/mvt/ios/modules/fs/contacts.py index ac99842..cea082d 100644 --- a/mvt/ios/modules/fs/contacts.py +++ b/mvt/ios/modules/fs/contacts.py @@ -1,7 +1,7 @@ # Mobile Verification Toolkit (MVT) -# Copyright (c) 2021 MVT Project Developers. -# See the file 'LICENSE' for usage and copying permissions, or find a copy at -# https://github.com/mvt-project/mvt/blob/main/LICENSE +# Copyright (c) 2021 The MVT Project Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ import sqlite3 diff --git a/mvt/ios/modules/fs/filesystem.py b/mvt/ios/modules/fs/filesystem.py index fc170f6..692ee58 100644 --- a/mvt/ios/modules/fs/filesystem.py +++ b/mvt/ios/modules/fs/filesystem.py @@ -1,15 +1,16 @@ # Mobile Verification Toolkit (MVT) -# Copyright (c) 2021 MVT Project Developers. -# See the file 'LICENSE' for usage and copying permissions, or find a copy at -# https://github.com/mvt-project/mvt/blob/main/LICENSE +# Copyright (c) 2021 The MVT Project Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ -import os import datetime +import os from mvt.common.utils import convert_timestamp_to_iso from .base import IOSExtraction + class Filesystem(IOSExtraction): """This module extracts creation and modification date of files from a full file-system dump.""" diff --git a/mvt/ios/modules/fs/firefox_favicon.py b/mvt/ios/modules/fs/firefox_favicon.py index ad0c09d..f946576 100644 --- a/mvt/ios/modules/fs/firefox_favicon.py +++ b/mvt/ios/modules/fs/firefox_favicon.py @@ -1,11 +1,11 @@ # Mobile Verification Toolkit (MVT) -# Copyright (c) 2021 MVT Project Developers. -# See the file 'LICENSE' for usage and copying permissions, or find a copy at -# https://github.com/mvt-project/mvt/blob/main/LICENSE +# Copyright (c) 2021 The MVT Project Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ import sqlite3 - from datetime import datetime + from mvt.common.url import URL from mvt.common.utils import convert_mactime_to_unix, convert_timestamp_to_iso diff --git a/mvt/ios/modules/fs/firefox_history.py b/mvt/ios/modules/fs/firefox_history.py index 8d9bff9..6690747 100644 --- a/mvt/ios/modules/fs/firefox_history.py +++ b/mvt/ios/modules/fs/firefox_history.py @@ -1,11 +1,11 @@ # Mobile Verification Toolkit (MVT) -# Copyright (c) 2021 MVT Project Developers. -# See the file 'LICENSE' for usage and copying permissions, or find a copy at -# https://github.com/mvt-project/mvt/blob/main/LICENSE +# Copyright (c) 2021 The MVT Project Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ import sqlite3 - from datetime import datetime + from mvt.common.url import URL from mvt.common.utils import convert_mactime_to_unix, convert_timestamp_to_iso diff --git a/mvt/ios/modules/fs/idstatuscache.py b/mvt/ios/modules/fs/idstatuscache.py index b447e58..cb230e3 100644 --- a/mvt/ios/modules/fs/idstatuscache.py +++ b/mvt/ios/modules/fs/idstatuscache.py @@ -1,12 +1,13 @@ # Mobile Verification Toolkit (MVT) -# Copyright (c) 2021 MVT Project Developers. -# See the file 'LICENSE' for usage and copying permissions, or find a copy at -# https://github.com/mvt-project/mvt/blob/main/LICENSE +# Copyright (c) 2021 The MVT Project Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ -import os -import glob -import biplist import collections +import glob +import os + +import biplist from mvt.common.utils import convert_mactime_to_unix, convert_timestamp_to_iso diff --git a/mvt/ios/modules/fs/interactionc.py b/mvt/ios/modules/fs/interactionc.py index a685a93..a406d10 100644 --- a/mvt/ios/modules/fs/interactionc.py +++ b/mvt/ios/modules/fs/interactionc.py @@ -1,7 +1,7 @@ # Mobile Verification Toolkit (MVT) -# Copyright (c) 2021 MVT Project Developers. -# See the file 'LICENSE' for usage and copying permissions, or find a copy at -# https://github.com/mvt-project/mvt/blob/main/LICENSE +# Copyright (c) 2021 The MVT Project Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ import sqlite3 from base64 import b64encode diff --git a/mvt/ios/modules/fs/locationd.py b/mvt/ios/modules/fs/locationd.py index fe39f17..700be01 100644 --- a/mvt/ios/modules/fs/locationd.py +++ b/mvt/ios/modules/fs/locationd.py @@ -1,10 +1,11 @@ # Mobile Verification Toolkit (MVT) -# Copyright (c) 2021 MVT Project Developers. -# See the file 'LICENSE' for usage and copying permissions, or find a copy at -# https://github.com/mvt-project/mvt/blob/main/LICENSE +# Copyright (c) 2021 The MVT Project Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ -import os import glob +import os + import biplist from mvt.common.utils import convert_mactime_to_unix, convert_timestamp_to_iso @@ -43,10 +44,10 @@ class LocationdClients(IOSExtraction): for ts in self.timestamps: if ts in record.keys(): records.append({ - "timestamp": entry[ts], + "timestamp": record[ts], "module": self.__class__.__name__, "event": ts, - "data": f"{ts} from {entry['package']}" + "data": f"{ts} from {record['package']}" }) return records @@ -62,7 +63,7 @@ class LocationdClients(IOSExtraction): result["package"] = app for ts in self.timestamps: if ts in result.keys(): - result[ts] = convert_timestamp_to_iso(convert_mactime_to_unix(result[date])) + result[ts] = convert_timestamp_to_iso(convert_mactime_to_unix(result[ts])) self.results.append(result) diff --git a/mvt/ios/modules/fs/manifest.py b/mvt/ios/modules/fs/manifest.py index d5c1c74..8f230bc 100644 --- a/mvt/ios/modules/fs/manifest.py +++ b/mvt/ios/modules/fs/manifest.py @@ -1,18 +1,21 @@ # Mobile Verification Toolkit (MVT) -# Copyright (c) 2021 MVT Project Developers. -# See the file 'LICENSE' for usage and copying permissions, or find a copy at -# https://github.com/mvt-project/mvt/blob/main/LICENSE +# Copyright (c) 2021 The MVT Project Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ +import datetime import io import os -import biplist import sqlite3 -import datetime +import biplist + +from mvt.common.module import DatabaseNotFoundError from mvt.common.utils import convert_timestamp_to_iso from .base import IOSExtraction + class Manifest(IOSExtraction): """This module extracts information from a backup Manifest.db file.""" @@ -40,6 +43,8 @@ class Manifest(IOSExtraction): def serialize(self, record): records = [] + if "modified" not in record or "statusChanged" not in record: + return for ts in set([record["created"], record["modified"], record["statusChanged"]]): macb = "" macb += "M" if ts == record["modified"] else "-" @@ -63,12 +68,15 @@ class Manifest(IOSExtraction): for result in self.results: if not "relativePath" in result: continue - - if os.path.basename(result["relativePath"]) == "com.apple.CrashReporter.plist" and result["domain"] == "RootDomain": - self.log.warning("Found a potentially suspicious \"com.apple.CrashReporter.plist\" file created in RootDomain") - self.detected.append(result) + if not result["relativePath"]: continue + if result["domain"]: + if os.path.basename(result["relativePath"]) == "com.apple.CrashReporter.plist" and result["domain"] == "RootDomain": + self.log.warning("Found a potentially suspicious \"com.apple.CrashReporter.plist\" file created in RootDomain") + self.detected.append(result) + continue + if self.indicators.check_file(result["relativePath"]): self.log.warning("Found a known malicious file at path: %s", result["relativePath"]) self.detected.append(result) @@ -84,7 +92,7 @@ class Manifest(IOSExtraction): def run(self): manifest_db_path = os.path.join(self.base_folder, "Manifest.db") if not os.path.isfile(manifest_db_path): - raise FileNotFoundError("Impossible to find the module's database file") + raise DatabaseNotFoundError("Impossible to find the module's database file") self.log.info("Found Manifest.db database at path: %s", manifest_db_path) diff --git a/mvt/ios/modules/fs/net_base.py b/mvt/ios/modules/fs/net_base.py index b5cd550..e1ab93b 100644 --- a/mvt/ios/modules/fs/net_base.py +++ b/mvt/ios/modules/fs/net_base.py @@ -1,16 +1,17 @@ # Mobile Verification Toolkit (MVT) -# Copyright (c) 2021 MVT Project Developers. -# See the file 'LICENSE' for usage and copying permissions, or find a copy at -# https://github.com/mvt-project/mvt/blob/main/LICENSE +# Copyright (c) 2021 The MVT Project Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ -import sqlite3 import operator +import sqlite3 from pathlib import Path from mvt.common.utils import convert_mactime_to_unix, convert_timestamp_to_iso from .base import IOSExtraction + class NetBase(IOSExtraction): """This class provides a base for DataUsage and NetUsage extraction modules.""" @@ -167,6 +168,10 @@ class NetBase(IOSExtraction): results_by_proc = {proc["proc_id"]: proc for proc in self.results if proc["proc_id"]} all_proc_id = sorted(results_by_proc.keys()) + # Fix issue #108 + if not all_proc_id: + return + missing_procs, last_proc_id = {}, None for proc_id in range(min(all_proc_id), max(all_proc_id)): if proc_id not in all_proc_id: diff --git a/mvt/ios/modules/fs/net_datausage.py b/mvt/ios/modules/fs/net_datausage.py index dc9d319..633bffc 100644 --- a/mvt/ios/modules/fs/net_datausage.py +++ b/mvt/ios/modules/fs/net_datausage.py @@ -1,7 +1,7 @@ # Mobile Verification Toolkit (MVT) -# Copyright (c) 2021 MVT Project Developers. -# See the file 'LICENSE' for usage and copying permissions, or find a copy at -# https://github.com/mvt-project/mvt/blob/main/LICENSE +# Copyright (c) 2021 The MVT Project Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ from .net_base import NetBase diff --git a/mvt/ios/modules/fs/net_netusage.py b/mvt/ios/modules/fs/net_netusage.py index 5f0e853..72bd55f 100644 --- a/mvt/ios/modules/fs/net_netusage.py +++ b/mvt/ios/modules/fs/net_netusage.py @@ -1,7 +1,7 @@ # Mobile Verification Toolkit (MVT) -# Copyright (c) 2021 MVT Project Developers. -# See the file 'LICENSE' for usage and copying permissions, or find a copy at -# https://github.com/mvt-project/mvt/blob/main/LICENSE +# Copyright (c) 2021 The MVT Project Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ from .net_base import NetBase diff --git a/mvt/ios/modules/fs/safari_browserstate.py b/mvt/ios/modules/fs/safari_browserstate.py index 98d6bcb..0836185 100644 --- a/mvt/ios/modules/fs/safari_browserstate.py +++ b/mvt/ios/modules/fs/safari_browserstate.py @@ -1,14 +1,15 @@ # Mobile Verification Toolkit (MVT) -# Copyright (c) 2021 MVT Project Developers. -# See the file 'LICENSE' for usage and copying permissions, or find a copy at -# https://github.com/mvt-project/mvt/blob/main/LICENSE +# Copyright (c) 2021 The MVT Project Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ import io -import biplist import sqlite3 -from mvt.common.utils import convert_mactime_to_unix, convert_timestamp_to_iso -from mvt.common.utils import keys_bytes_to_string +import biplist + +from mvt.common.utils import (convert_mactime_to_unix, + convert_timestamp_to_iso, keys_bytes_to_string) from .base import IOSExtraction diff --git a/mvt/ios/modules/fs/safari_favicon.py b/mvt/ios/modules/fs/safari_favicon.py index 3bd5c36..0a931ef 100644 --- a/mvt/ios/modules/fs/safari_favicon.py +++ b/mvt/ios/modules/fs/safari_favicon.py @@ -1,7 +1,7 @@ # Mobile Verification Toolkit (MVT) -# Copyright (c) 2021 MVT Project Developers. -# See the file 'LICENSE' for usage and copying permissions, or find a copy at -# https://github.com/mvt-project/mvt/blob/main/LICENSE +# Copyright (c) 2021 The MVT Project Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ import sqlite3 diff --git a/mvt/ios/modules/fs/safari_history.py b/mvt/ios/modules/fs/safari_history.py index 4c498a9..487ac5a 100644 --- a/mvt/ios/modules/fs/safari_history.py +++ b/mvt/ios/modules/fs/safari_history.py @@ -1,7 +1,7 @@ # Mobile Verification Toolkit (MVT) -# Copyright (c) 2021 MVT Project Developers. -# See the file 'LICENSE' for usage and copying permissions, or find a copy at -# https://github.com/mvt-project/mvt/blob/main/LICENSE +# Copyright (c) 2021 The MVT Project Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ import sqlite3 diff --git a/mvt/ios/modules/fs/sms.py b/mvt/ios/modules/fs/sms.py index 1208074..313f57d 100644 --- a/mvt/ios/modules/fs/sms.py +++ b/mvt/ios/modules/fs/sms.py @@ -1,13 +1,13 @@ # Mobile Verification Toolkit (MVT) -# Copyright (c) 2021 MVT Project Developers. -# See the file 'LICENSE' for usage and copying permissions, or find a copy at -# https://github.com/mvt-project/mvt/blob/main/LICENSE +# Copyright (c) 2021 The MVT Project Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ import sqlite3 from base64 import b64encode -from mvt.common.utils import check_for_links -from mvt.common.utils import convert_mactime_to_unix, convert_timestamp_to_iso +from mvt.common.utils import (check_for_links, convert_mactime_to_unix, + convert_timestamp_to_iso) from .base import IOSExtraction diff --git a/mvt/ios/modules/fs/sms_attachments.py b/mvt/ios/modules/fs/sms_attachments.py index 57cb8c1..8835e88 100644 --- a/mvt/ios/modules/fs/sms_attachments.py +++ b/mvt/ios/modules/fs/sms_attachments.py @@ -1,13 +1,13 @@ # Mobile Verification Toolkit (MVT) -# Copyright (c) 2021 MVT Project Developers. -# See the file 'LICENSE' for usage and copying permissions, or find a copy at -# https://github.com/mvt-project/mvt/blob/main/LICENSE +# Copyright (c) 2021 The MVT Project Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ import sqlite3 from base64 import b64encode -from mvt.common.utils import check_for_links -from mvt.common.utils import convert_mactime_to_unix, convert_timestamp_to_iso +from mvt.common.utils import (check_for_links, convert_mactime_to_unix, + convert_timestamp_to_iso) from .base import IOSExtraction diff --git a/mvt/ios/modules/fs/version_history.py b/mvt/ios/modules/fs/version_history.py index 70dc054..1b091de 100644 --- a/mvt/ios/modules/fs/version_history.py +++ b/mvt/ios/modules/fs/version_history.py @@ -1,10 +1,10 @@ # Mobile Verification Toolkit (MVT) -# Copyright (c) 2021 MVT Project Developers. -# See the file 'LICENSE' for usage and copying permissions, or find a copy at -# https://github.com/mvt-project/mvt/blob/main/LICENSE +# Copyright (c) 2021 The MVT Project Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ -import json import datetime +import json from mvt.common.utils import convert_timestamp_to_iso diff --git a/mvt/ios/modules/fs/webkit_base.py b/mvt/ios/modules/fs/webkit_base.py index 2adb717..75979b3 100644 --- a/mvt/ios/modules/fs/webkit_base.py +++ b/mvt/ios/modules/fs/webkit_base.py @@ -1,14 +1,15 @@ # Mobile Verification Toolkit (MVT) -# Copyright (c) 2021 MVT Project Developers. -# See the file 'LICENSE' for usage and copying permissions, or find a copy at -# https://github.com/mvt-project/mvt/blob/main/LICENSE +# Copyright (c) 2021 The MVT Project Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ -import os import datetime +import os + +from mvt.common.utils import convert_timestamp_to_iso from .base import IOSExtraction -from mvt.common.utils import convert_timestamp_to_iso class WebkitBase(IOSExtraction): """This class is a base for other WebKit-related modules.""" diff --git a/mvt/ios/modules/fs/webkit_indexeddb.py b/mvt/ios/modules/fs/webkit_indexeddb.py index 55d65ec..a4aa2a0 100644 --- a/mvt/ios/modules/fs/webkit_indexeddb.py +++ b/mvt/ios/modules/fs/webkit_indexeddb.py @@ -1,7 +1,7 @@ # Mobile Verification Toolkit (MVT) -# Copyright (c) 2021 MVT Project Developers. -# See the file 'LICENSE' for usage and copying permissions, or find a copy at -# https://github.com/mvt-project/mvt/blob/main/LICENSE +# Copyright (c) 2021 The MVT Project Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ from .webkit_base import WebkitBase diff --git a/mvt/ios/modules/fs/webkit_localstorage.py b/mvt/ios/modules/fs/webkit_localstorage.py index 8b4fddb..868b601 100644 --- a/mvt/ios/modules/fs/webkit_localstorage.py +++ b/mvt/ios/modules/fs/webkit_localstorage.py @@ -1,7 +1,7 @@ # Mobile Verification Toolkit (MVT) -# Copyright (c) 2021 MVT Project Developers. -# See the file 'LICENSE' for usage and copying permissions, or find a copy at -# https://github.com/mvt-project/mvt/blob/main/LICENSE +# Copyright (c) 2021 The MVT Project Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ from .webkit_base import WebkitBase diff --git a/mvt/ios/modules/fs/webkit_resource_load_statistics.py b/mvt/ios/modules/fs/webkit_resource_load_statistics.py new file mode 100644 index 0000000..2fe4a03 --- /dev/null +++ b/mvt/ios/modules/fs/webkit_resource_load_statistics.py @@ -0,0 +1,98 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021 The MVT Project Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +import datetime +import os +import sqlite3 + +from mvt.common.utils import convert_mactime_to_unix, convert_timestamp_to_iso + +from .base import IOSExtraction + +WEBKIT_RESOURCELOADSTATICS_BACKUP_RELPATH = "Library/WebKit/WebsiteData/ResourceLoadStatistics/observations.db" +WEBKIT_RESOURCELOADSTATICS_ROOT_PATHS = [ + "private/var/mobile/Containers/Data/Application/*/Library/WebKit/WebsiteData/ResourceLoadStatistics/observations.db", + "private/var/mobile/Containers/Data/Application/*/SystemData/com.apple.SafariViewService/Library/WebKit/WebsiteData/observations.db", +] + +class WebkitResourceLoadStatistics(IOSExtraction): + """This module extracts records from WebKit ResourceLoadStatistics observations.db. + """ + # TODO: Add serialize(). + + def __init__(self, file_path=None, base_folder=None, output_folder=None, + fast_mode=False, log=None, results=[]): + super().__init__(file_path=file_path, base_folder=base_folder, + output_folder=output_folder, fast_mode=fast_mode, + log=log, results=results) + + def check_indicators(self): + if not self.indicators: + return + + self.detected = {} + for key, items in self.results.items(): + for item in items: + if self.indicators.check_domain(item["registrable_domain"]): + if key not in self.detected: + self.detected[key] = [item,] + else: + self.detected[key].append(item) + + def _process_observations_db(self, db_path, key): + self.log.info("Found WebKit ResourceLoadStatistics observations.db file at path %s", db_path) + + if self._is_database_malformed(db_path): + self._recover_database(db_path) + + conn = sqlite3.connect(db_path) + cur = conn.cursor() + + try: + cur.execute("SELECT * from ObservedDomains;") + except sqlite3.OperationalError: + return + + if not key in self.results: + self.results[key] = [] + + for row in cur: + self.results[key].append(dict( + domain_id=row[0], + registrable_domain=row[1], + last_seen=row[2], + had_user_interaction=bool(row[3]), + # TODO: Fix isodate. + last_seen_isodate=convert_timestamp_to_iso(datetime.datetime.utcfromtimestamp(int(row[2]))), + )) + + if len(self.results[key]) > 0: + self.log.info("Extracted a total of %d records from %s", len(self.results[key]), db_path) + + def run(self): + self.results = {} + + if self.is_backup: + manifest_db_path = os.path.join(self.base_folder, "Manifest.db") + if not os.path.exists(manifest_db_path): + self.log.info("Unable to search for WebKit observations.db files in backup because of missing Manifest.db") + return + + try: + conn = sqlite3.connect(manifest_db_path) + cur = conn.cursor() + cur.execute("SELECT fileID, domain FROM Files WHERE relativePath = ?;", (WEBKIT_RESOURCELOADSTATICS_BACKUP_RELPATH,)) + except Exception as e: + self.log.error("Unable to search for WebKit observations.db files in backup because of failed query to Manifest.db: %s", e) + + for row in cur: + file_id = row[0] + domain = row[1] + db_path = os.path.join(self.base_folder, file_id[0:2], file_id) + if os.path.exists(db_path): + self._process_observations_db(db_path=db_path, key=f"{domain}/{WEBKIT_RESOURCELOADSTATICS_BACKUP_RELPATH}") + elif self.is_fs_dump: + for db_path in self._find_paths(WEBKIT_RESOURCELOADSTATICS_ROOT_PATHS): + self._process_observations_db(db_path=db_path, key=os.path.relpath(db_path, self.base_folder)) diff --git a/mvt/ios/modules/fs/webkit_safariviewservice.py b/mvt/ios/modules/fs/webkit_safariviewservice.py index 33f993c..c0b5a82 100644 --- a/mvt/ios/modules/fs/webkit_safariviewservice.py +++ b/mvt/ios/modules/fs/webkit_safariviewservice.py @@ -1,7 +1,7 @@ # Mobile Verification Toolkit (MVT) -# Copyright (c) 2021 MVT Project Developers. -# See the file 'LICENSE' for usage and copying permissions, or find a copy at -# https://github.com/mvt-project/mvt/blob/main/LICENSE +# Copyright (c) 2021 The MVT Project Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ from .webkit_base import WebkitBase diff --git a/mvt/ios/modules/fs/webkit_session_resource_log.py b/mvt/ios/modules/fs/webkit_session_resource_log.py index 299306d..c7b5c2f 100644 --- a/mvt/ios/modules/fs/webkit_session_resource_log.py +++ b/mvt/ios/modules/fs/webkit_session_resource_log.py @@ -1,10 +1,11 @@ # Mobile Verification Toolkit (MVT) -# Copyright (c) 2021 MVT Project Developers. -# See the file 'LICENSE' for usage and copying permissions, or find a copy at -# https://github.com/mvt-project/mvt/blob/main/LICENSE +# Copyright (c) 2021 The MVT Project Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ -import os import glob +import os + import biplist from mvt.common.utils import convert_timestamp_to_iso @@ -70,6 +71,9 @@ class WebkitSessionResourceLog(IOSExtraction): return domains def check_indicators(self): + if not self.indicators: + return + for key, entries in self.results.items(): for entry in entries: source_domains = self._extract_domains(entry["redirect_source"]) diff --git a/mvt/ios/modules/fs/whatsapp.py b/mvt/ios/modules/fs/whatsapp.py index 34db73a..8e973df 100644 --- a/mvt/ios/modules/fs/whatsapp.py +++ b/mvt/ios/modules/fs/whatsapp.py @@ -1,12 +1,13 @@ # Mobile Verification Toolkit (MVT) -# Copyright (c) 2021 MVT Project Developers. -# See the file 'LICENSE' for usage and copying permissions, or find a copy at -# https://github.com/mvt-project/mvt/blob/main/LICENSE +# Copyright (c) 2021 The MVT Project Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ -import sqlite3 import logging +import sqlite3 -from mvt.common.utils import convert_mactime_to_unix, convert_timestamp_to_iso, check_for_links +from mvt.common.utils import (check_for_links, convert_mactime_to_unix, + convert_timestamp_to_iso) from .base import IOSExtraction @@ -73,7 +74,7 @@ class Whatsapp(IOSExtraction): # Extract links from the WhatsApp message. message_links = check_for_links(new_message["ZTEXT"]) - # If we find mesages, or if there's an empty message we add it to the list. + # If we find messages, or if there's an empty message we add it to the list. if new_message["ZTEXT"] and (message_links or new_message["ZTEXT"].strip() == ""): self.results.append(new_message) diff --git a/mvt/ios/versions.py b/mvt/ios/versions.py index 6ca0b50..fc75a33 100644 --- a/mvt/ios/versions.py +++ b/mvt/ios/versions.py @@ -1,190 +1,234 @@ # Mobile Verification Toolkit (MVT) -# Copyright (c) 2021 MVT Project Developers. -# See the file 'LICENSE' for usage and copying permissions, or find a copy at -# https://github.com/mvt-project/mvt/blob/main/LICENSE +# Copyright (c) 2021 The MVT Project Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +IPHONE_MODELS = [ + {"description": "iPhone 4S", "identifier": "iPhone4,1"}, + {"description": "iPhone 5", "identifier": "iPhone5,1"}, + {"description": "iPhone 5", "identifier": "iPhone5,2"}, + {"description": "iPhone 5c", "identifier": "iPhone5,3"}, + {"description": "iPhone 5c", "identifier": "iPhone5,4"}, + {"description": "iPhone 5s", "identifier": "iPhone6,1"}, + {"description": "iPhone 5s", "identifier": "iPhone6,2"}, + {"description": "iPhone 6 Plus", "identifier": "iPhone7,1"}, + {"description": "iPhone 6", "identifier": "iPhone7,2"}, + {"description": "iPhone 6s", "identifier": "iPhone8,1"}, + {"description": "iPhone 6s Plus", "identifier": "iPhone8,2"}, + {"description": "iPhone SE (1st generation)", "identifier": "iPhone8,4"}, + {"description": "iPhone 7", "identifier": "iPhone9,1"}, + {"description": "iPhone 7 Plus", "identifier": "iPhone9,2"}, + {"description": "iPhone 7", "identifier": "iPhone9,3"}, + {"description": "iPhone 7 Plus", "identifier": "iPhone9,4"}, + {"description": "iPhone 8", "identifier": "iPhone10,1"}, + {"description": "iPhone 8 Plus", "identifier": "iPhone10,2"}, + {"description": "iPhone X", "identifier": "iPhone10,3"}, + {"description": "iPhone 8", "identifier": "iPhone10,4"}, + {"description": "iPhone 8 Plus", "identifier": "iPhone10,5"}, + {"description": "iPhone X", "identifier": "iPhone10,6"}, + {"description": "iPhone XS", "identifier": "iPhone11,2"}, + {"description": "iPhone XS Max", "identifier": "iPhone11,4"}, + {"description": "iPhone XS Max", "identifier": "iPhone11,6"}, + {"description": "iPhone XR", "identifier": "iPhone11,8"}, + {"description": "iPhone 11", "identifier": "iPhone12,1"}, + {"description": "iPhone 11 Pro", "identifier": "iPhone12,3"}, + {"description": "iPhone 11 Pro Max", "identifier": "iPhone12,5"}, + {"description": "iPhone SE (2nd generation)", "identifier": "iPhone12,8"}, + {"description": "iPhone 12 mini", "identifier": "iPhone13,1"}, + {"description": "iPhone 12", "identifier": "iPhone13,2"}, + {"description": "iPhone 12 Pro", "identifier": "iPhone13,3"}, + {"description": "iPhone 12 Pro Max", "identifier": "iPhone13,4"}, +] IPHONE_IOS_VERSIONS = [ - {"build": "7E18", "version": "3.1.3"}, - {"build": "7D11", "version": "3.1.2"}, - {"build": "7C144", "version": "3.1"}, - {"build": "7A400", "version": "3.0.1"}, - {"build": "7A341", "version": "3.0"}, - {"build": "5H11", "version": "2.2.1"}, - {"build": "5G77", "version": "2.2"}, - {"build": "5F136", "version": "2.1"}, - {"build": "5C1", "version": "2.0.2"}, - {"build": "5B108", "version": "2.0.1"}, - {"build": "5A347", "version": "2.0"}, - {"build": "4A102", "version": "1.1.4"}, - {"build": "4A93", "version": "1.1.3"}, - {"build": "3B48b", "version": "1.1.2"}, - {"build": "3A109a", "version": "1.1.1"}, - {"build": "1C28", "version": "1.0.2"}, - {"build": "1C25", "version": "1.0.1"}, {"build": "1A543a", "version": "1.0"}, - {"build": "8C148", "version": "4.2"}, - {"build": "8B117", "version": "4.1"}, - {"build": "8A306", "version": "4.0.1"}, + {"build": "1C25", "version": "1.0.1"}, + {"build": "1C28", "version": "1.0.2"}, + {"build": "3A109a", "version": "1.1.1"}, + {"build": "3B48b", "version": "1.1.2"}, + {"build": "4A93", "version": "1.1.3"}, + {"build": "4A102", "version": "1.1.4"}, + {"build": "5A347", "version": "2.0"}, + {"build": "5B108", "version": "2.0.1"}, + {"build": "5C1", "version": "2.0.2"}, + {"build": "5F136", "version": "2.1"}, + {"build": "5G77", "version": "2.2"}, + {"build": "5H11", "version": "2.2.1"}, + {"build": "7A341", "version": "3.0"}, + {"build": "7A400", "version": "3.0.1"}, + {"build": "7C144", "version": "3.1"}, + {"build": "7D11", "version": "3.1.2"}, + {"build": "7E18", "version": "3.1.3"}, {"build": "8A293", "version": "4.0"}, - {"build": "10B500", "version": "6.1.6"}, - {"build": "10B329", "version": "6.1.3"}, - {"build": "10B146", "version": "6.1.2"}, - {"build": "10B141", "version": "6.1"}, - {"build": "10A523", "version": "6.0.1"}, - {"build": "10A403", "version": "6.0"}, - {"build": "9B206", "version": "5.1.1"}, - {"build": "9B176", "version": "5.1"}, - {"build": "9A405", "version": "5.0.1"}, - {"build": "9A334", "version": "5.0"}, - {"build": "8L1", "version": "4.3.5"}, - {"build": "8K2", "version": "4.3.4"}, - {"build": "8J2", "version": "4.3.3"}, - {"build": "8F190", "version": "4.3"}, + {"build": "8A306", "version": "4.0.1"}, + {"build": "8B117", "version": "4.1"}, + {"build": "8C148", "version": "4.2"}, {"build": "8C148a", "version": "4.2.1"}, - {"build": "11D257", "version": "7.1.2"}, - {"build": "11D201", "version": "7.1.1"}, - {"build": "11D169", "version": "7.1"}, - {"build": "11B651", "version": "7.0.6"}, - {"build": "11B554a", "version": "7.0.4"}, - {"build": "11B511", "version": "7.0.3"}, - {"build": "10B144", "version": "6.1"}, - {"build": "9B208", "version": "5.1.1"}, {"build": "8C148", "version": "4.2.1"}, - {"build": "11D167", "version": "7.1"}, {"build": "8E600", "version": "4.2.10"}, - {"build": "8E501", "version": "4.2.9"}, {"build": "8E401", "version": "4.2.8"}, - {"build": "13G37", "version": "9.3.6"}, - {"build": "13G36", "version": "9.3.5"}, - {"build": "13G35", "version": "9.3.4"}, - {"build": "13G34", "version": "9.3.3"}, - {"build": "13F69", "version": "9.3.2"}, - {"build": "13E238", "version": "9.3.1"}, + {"build": "8E501", "version": "4.2.9"}, + {"build": "8F190", "version": "4.3"}, + {"build": "8J2", "version": "4.3.3"}, + {"build": "8K2", "version": "4.3.4"}, + {"build": "8L1", "version": "4.3.5"}, + {"build": "9A334", "version": "5.0"}, + {"build": "9A405", "version": "5.0.1"}, + {"build": "9A406", "version": "5.0.1"}, + {"build": "9B176", "version": "5.1"}, + {"build": "9B179", "version": "5.1"}, + {"build": "9B206", "version": "5.1.1"}, + {"build": "9B208", "version": "5.1.1"}, + {"build": "10A403", "version": "6.0"}, + {"build": "10A405", "version": "6.0"}, + {"build": "10A523", "version": "6.0.1"}, + {"build": "10A525", "version": "6.0.1"}, + {"build": "10A551", "version": "6.0.2"}, + {"build": "10B141", "version": "6.1"}, + {"build": "10B144", "version": "6.1"}, + {"build": "10B142", "version": "6.1"}, + {"build": "10B143", "version": "6.1"}, + {"build": "10B145", "version": "6.1.1"}, + {"build": "10B146", "version": "6.1.2"}, + {"build": "10B329", "version": "6.1.3"}, + {"build": "10B350", "version": "6.1.4"}, + {"build": "10B500", "version": "6.1.6"}, + {"build": "11B511", "version": "7.0.3"}, + {"build": "11B554a", "version": "7.0.4"}, + {"build": "11B601", "version": "7.0.5"}, + {"build": "11B651", "version": "7.0.6"}, + {"build": "11D169", "version": "7.1"}, + {"build": "11D167", "version": "7.1"}, + {"build": "11D201", "version": "7.1.1"}, + {"build": "11D257", "version": "7.1.2"}, + {"build": "12A365", "version": "8.0"}, + {"build": "12A366", "version": "8.0"}, + {"build": "12A402", "version": "8.0.1"}, + {"build": "12A405", "version": "8.0.2"}, + {"build": "12B411", "version": "8.1"}, + {"build": "12B435", "version": "8.1.1"}, + {"build": "12B436", "version": "8.1.1"}, + {"build": "12B440", "version": "8.1.2"}, + {"build": "12B466", "version": "8.1.3"}, + {"build": "12D508", "version": "8.2"}, + {"build": "12F70", "version": "8.3"}, + {"build": "12H143", "version": "8.4"}, + {"build": "12H321", "version": "8.4.1"}, + {"build": "13A344", "version": "9.0"}, + {"build": "13A342", "version": "9.0"}, + {"build": "13A343", "version": "9.0"}, + {"build": "13A404", "version": "9.0.1"}, + {"build": "13A405", "version": "9.0.1"}, + {"build": "13A452", "version": "9.0.2"}, + {"build": "13B143", "version": "9.1"}, + {"build": "13C75", "version": "9.2"}, + {"build": "13D15", "version": "9.2.1"}, + {"build": "13D20", "version": "9.2.1"}, {"build": "13E237", "version": "9.3"}, {"build": "13E233", "version": "9.3"}, - {"build": "13D15", "version": "9.2.1"}, - {"build": "13C75", "version": "9.2"}, - {"build": "13B143", "version": "9.1"}, - {"build": "13A452", "version": "9.0.2"}, - {"build": "13A404", "version": "9.0.1"}, - {"build": "13A344", "version": "9.0"}, - {"build": "12H321", "version": "8.4.1"}, - {"build": "12H143", "version": "8.4"}, - {"build": "12F70", "version": "8.3"}, - {"build": "12D508", "version": "8.2"}, - {"build": "12B466", "version": "8.1.3"}, - {"build": "12B440", "version": "8.1.2"}, - {"build": "12B435", "version": "8.1.1"}, - {"build": "12B411", "version": "8.1"}, - {"build": "12A405", "version": "8.0.2"}, - {"build": "12A402", "version": "8.0.1"}, - {"build": "12A365", "version": "8.0"}, - {"build": "10B145", "version": "6.1.1"}, - {"build": "10B142", "version": "6.1"}, - {"build": "9B179", "version": "5.1"}, - {"build": "9A406", "version": "5.0.1"}, - {"build": "14G61", "version": "10.3.4"}, - {"build": "14G60", "version": "10.3.3"}, - {"build": "14F89", "version": "10.3.2"}, - {"build": "14E304", "version": "10.3.1"}, - {"build": "14E277", "version": "10.3"}, - {"build": "14D27", "version": "10.2.1"}, - {"build": "14C92", "version": "10.2"}, + {"build": "13E234", "version": "9.3"}, + {"build": "13E238", "version": "9.3.1"}, + {"build": "13F69", "version": "9.3.2"}, + {"build": "13G34", "version": "9.3.3"}, + {"build": "13G35", "version": "9.3.4"}, + {"build": "13G36", "version": "9.3.5"}, + {"build": "13G37", "version": "9.3.6"}, + {"build": "14A403", "version": "10.0.1"}, + {"build": "14A456", "version": "10.0.2"}, + {"build": "14A551", "version": "10.0.3"}, + {"build": "14B72", "version": "10.1"}, + {"build": "14B72c", "version": "10.1"}, {"build": "14B150", "version": "10.1.1"}, {"build": "14B100", "version": "10.1.1"}, - {"build": "14B72", "version": "10.1"}, - {"build": "14A456", "version": "10.0.2"}, - {"build": "14A403", "version": "10.0.1"}, - {"build": "10B350", "version": "6.1.4"}, - {"build": "10B143", "version": "6.1"}, - {"build": "10A551", "version": "6.0.2"}, - {"build": "10A525", "version": "6.0.1"}, - {"build": "10A405", "version": "6.0"}, - {"build": "11B601", "version": "7.0.5"}, - {"build": "18F72", "version": "14.6"}, - {"build": "18E199", "version": "14.5"}, - {"build": "18E212", "version": "14.5.1"}, + {"build": "14C92", "version": "10.2"}, + {"build": "14D27", "version": "10.2.1"}, + {"build": "14E277", "version": "10.3"}, + {"build": "14E304", "version": "10.3.1"}, + {"build": "14F89", "version": "10.3.2"}, + {"build": "14G60", "version": "10.3.3"}, + {"build": "14G61", "version": "10.3.4"}, + {"build": "15A372", "version": "11.0"}, + {"build": "15A402", "version": "11.0.1"}, + {"build": "15A421", "version": "11.0.2"}, + {"build": "15A432", "version": "11.0.3"}, + {"build": "15B93", "version": "11.1"}, + {"build": "15B150", "version": "11.1.1"}, + {"build": "15B202", "version": "11.1.2"}, + {"build": "15C114", "version": "11.2"}, + {"build": "15C153", "version": "11.2.1"}, + {"build": "15C202", "version": "11.2.2"}, + {"build": "15D60", "version": "11.2.5"}, + {"build": "15D100", "version": "11.2.6"}, + {"build": "15E216", "version": "11.3"}, + {"build": "15E302", "version": "11.3.1"}, + {"build": "15F79", "version": "11.4"}, + {"build": "15G77", "version": "11.4.1"}, + {"build": "16A366", "version": "12.0"}, + {"build": "16A404", "version": "12.0.1"}, + {"build": "16A405", "version": "12.0.1"}, + {"build": "16B92", "version": "12.1"}, + {"build": "16B94", "version": "12.1"}, + {"build": "16B93", "version": "12.1"}, + {"build": "16C50", "version": "12.1.1"}, + {"build": "16C104", "version": "12.1.2"}, + {"build": "16C101", "version": "12.1.2"}, + {"build": "16D39", "version": "12.1.3"}, + {"build": "16D40", "version": "12.1.3"}, + {"build": "16D57", "version": "12.1.4"}, + {"build": "16E227", "version": "12.2"}, + {"build": "16F156", "version": "12.3"}, + {"build": "16F203", "version": "12.3.1"}, + {"build": "16F250", "version": "12.3.2"}, + {"build": "16G77", "version": "12.4"}, + {"build": "16G102", "version": "12.4.1"}, + {"build": "16G114", "version": "12.4.2"}, + {"build": "16G130", "version": "12.4.3"}, + {"build": "16G161", "version": "12.4.5"}, + {"build": "16G183", "version": "12.4.6"}, + {"build": "16G192", "version": "12.4.7"}, + {"build": "16G201", "version": "12.4.8"}, + {"build": "16H5", "version": "12.4.9"}, + {"build": "16H20", "version": "12.5"}, + {"build": "16H22", "version": "12.5.1"}, + {"build": "17A577", "version": "13.0"}, + {"build": "17A844", "version": "13.1"}, + {"build": "17A854", "version": "13.1.1"}, + {"build": "17A860", "version": "13.1.2"}, + {"build": "17A861", "version": "13.1.2"}, + {"build": "17A878", "version": "13.1.3"}, + {"build": "17B84", "version": "13.2"}, + {"build": "17B102", "version": "13.2.2"}, + {"build": "17B111", "version": "13.2.3"}, + {"build": "17C54", "version": "13.3"}, + {"build": "17D50", "version": "13.3.1"}, + {"build": "17E255", "version": "13.4"}, + {"build": "17E262", "version": "13.4.1"}, + {"build": "17E8258", "version": "13.4.1"}, + {"build": "17F75", "version": "13.5"}, + {"build": "17F80", "version": "13.5.1"}, + {"build": "17G68", "version": "13.6"}, + {"build": "17G80", "version": "13.6.1"}, + {"build": "17H35", "version": "13.7"}, + {"build": "18A373", "version": "14.0"}, + {"build": "18A393", "version": "14.0.1"}, + {"build": "18A8395", "version": "14.1"}, + {"build": "18B92", "version": "14.2"}, + {"build": "18C66", "version": "14.3"}, {"build": "18D52", "version": "14.4"}, {"build": "18D61", "version": "14.4.1"}, {"build": "18D70", "version": "14.4.2"}, - {"build": "18C66", "version": "14.3"}, - {"build": "18B92", "version": "14.2"}, - {"build": "18A8395", "version": "14.1"}, - {"build": "18A393", "version": "14.0.1"}, - {"build": "18A373", "version": "14.0"}, - {"build": "17H35", "version": "13.7"}, - {"build": "17G80", "version": "13.6.1"}, - {"build": "17G68", "version": "13.6"}, - {"build": "17F80", "version": "13.5.1"}, - {"build": "17F75", "version": "13.5"}, - {"build": "17E262", "version": "13.4.1"}, - {"build": "17E255", "version": "13.4"}, - {"build": "17C54", "version": "13.3"}, - {"build": "17D50", "version": "13.3.1"}, - {"build": "17B111", "version": "13.2.3"}, - {"build": "17B102", "version": "13.2.2"}, - {"build": "17B84", "version": "13.2"}, - {"build": "17A878", "version": "13.1.3"}, - {"build": "17A860", "version": "13.1.2"}, - {"build": "17A854", "version": "13.1.1"}, - {"build": "17A844", "version": "13.1"}, - {"build": "17A577", "version": "13.0"}, - {"build": "16H22", "version": "12.5.1"}, - {"build": "16H20", "version": "12.5"}, - {"build": "16H5", "version": "12.4.9"}, - {"build": "16G201", "version": "12.4.8"}, - {"build": "16G192", "version": "12.4.7"}, - {"build": "16G183", "version": "12.4.6"}, - {"build": "16G161", "version": "12.4.5"}, - {"build": "16G130", "version": "12.4.3"}, - {"build": "16G114", "version": "12.4.2"}, - {"build": "16G102", "version": "12.4.1"}, - {"build": "16G77", "version": "12.4"}, - {"build": "16F203", "version": "12.3.1"}, - {"build": "16F156", "version": "12.3"}, - {"build": "16E227", "version": "12.2"}, - {"build": "16D57", "version": "12.1.4"}, - {"build": "16D39", "version": "12.1.3"}, - {"build": "16C104", "version": "12.1.2"}, - {"build": "16C101", "version": "12.1.2"}, - {"build": "16C50", "version": "12.1.1"}, - {"build": "16B92", "version": "12.1"}, - {"build": "16A404", "version": "12.0.1"}, - {"build": "16A366", "version": "12.0"}, - {"build": "15G77", "version": "11.4.1"}, - {"build": "15F79", "version": "11.4"}, - {"build": "15E302", "version": "11.3.1"}, - {"build": "15E216", "version": "11.3"}, - {"build": "15D100", "version": "11.2.6"}, - {"build": "15D60", "version": "11.2.5"}, - {"build": "15C202", "version": "11.2.2"}, - {"build": "15C153", "version": "11.2.1"}, - {"build": "15C114", "version": "11.2"}, - {"build": "15B202", "version": "11.1.2"}, - {"build": "15B150", "version": "11.1.1"}, - {"build": "15B93", "version": "11.1"}, - {"build": "15A432", "version": "11.0.3"}, - {"build": "15A421", "version": "11.0.2"}, - {"build": "15A402", "version": "11.0.1"}, - {"build": "15A372", "version": "11.0"}, - {"build": "13D20", "version": "9.2.1"}, - {"build": "12B436", "version": "8.1.1"}, - {"build": "12A366", "version": "8.0"}, - {"build": "13E234", "version": "9.3"}, - {"build": "13A405", "version": "9.0.1"}, - {"build": "13A342", "version": "9.0"}, - {"build": "13A343", "version": "9.0"}, - {"build": "14B72c", "version": "10.1"}, - {"build": "14A551", "version": "10.0.3"}, - {"build": "16F250", "version": "12.3.2"}, - {"build": "17A861", "version": "13.1.2"}, - {"build": "16D40", "version": "12.1.3"}, - {"build": "16A405", "version": "12.0.1"}, - {"build": "16B94", "version": "12.1"}, - {"build": "16B93", "version": "12.1"}, - {"build": "17E8258", "version": "13.4.1"} + {"build": "18E199", "version": "14.5"}, + {"build": "18E212", "version": "14.5.1"}, + {"build": "18F72", "version": "14.6"}, + {"build": "18G69", "version": "14.7"}, + {"build": "18G82", "version": "14.7.1"}, ] +def get_device_desc_from_id(identifier, devices_list=IPHONE_MODELS): + for model in IPHONE_MODELS: + if identifier == model["identifier"]: + return model["description"] + def find_version_by_build(build): build = build.upper() for version in IPHONE_IOS_VERSIONS: diff --git a/setup.py b/setup.py index c705fad..621ef1d 100755 --- a/setup.py +++ b/setup.py @@ -1,13 +1,14 @@ # Mobile Verification Toolkit (MVT) -# Copyright (c) 2021 MVT Project Developers. -# See the file 'LICENSE' for usage and copying permissions, or find a copy at -# https://github.com/mvt-project/mvt/blob/main/LICENSE +# Copyright (c) 2021 The MVT Project Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ import os -from setuptools import setup, find_packages + +from setuptools import find_packages, setup __package_name__ = "mvt" -__version__ = "1.0.11" +__version__ = "1.0.17" __description__ = "Mobile Verification Toolkit" this_directory = os.path.abspath(os.path.dirname(__file__)) @@ -17,18 +18,18 @@ with open(readme_path, encoding="utf-8") as handle: requires = ( # Base dependencies: - "click", - "rich", - "tld", - "tqdm", - "requests", - "simplejson", + "click>=8.0.1", + "rich>=10.6.0", + "tld>=0.12.6", + "tqdm>=4.61.2", + "requests>=2.26.0", + "simplejson>=3.17.3", # iOS dependencies: - "biplist", - "iOSbackup", + "biplist>=1.0.3", + "iOSbackup>=0.9.912", # Android dependencies: - "adb-shell", - "libusb1", + "adb-shell>=0.4.0", + "libusb1>=1.9.3", ) def get_package_data(package):