commit 065a62cee1632a6e97564510886bb58397fec7bf Author: Nex Date: Fri Jul 16 08:05:01 2021 +0200 First commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..73b7def --- /dev/null +++ b/.gitignore @@ -0,0 +1,131 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ +*.pyc + diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..54123c1 --- /dev/null +++ b/LICENSE @@ -0,0 +1,401 @@ +MVT License 1.0 +=============== + +1. Definitions +-------------- + +1.1. "Contributor" + means each individual or legal entity that creates, contributes to + the creation of, or owns Covered Software. + +1.2. "Contributor Version" + means the combination of the Contributions of others (if any) used + by a Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + means Source Code Form to which the initial Contributor has attached + the notice in Exhibit A, the Executable Form of such Source Code + Form, and Modifications of such Source Code Form, in each case + including portions thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + (a) that the initial Contributor has attached the notice described + in Exhibit B to the Covered Software; or + + (b) that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the + terms of a Secondary License. + +1.6. "Executable Form" + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + means a work that combines Covered Software with other material, in + a separate file or files, that is not Covered Software. + +1.8. "License" + means this document. + +1.9. "Licensable" + means having the right to grant, to the maximum extent possible, + whether at the time of the initial grant or subsequently, any and + all of the rights conveyed by this License. + +1.10. "Modifications" + means any of the following: + + (a) any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered + Software; or + + (b) any new file in Source Code Form that contains any Covered + Software. + +1.11. "Patent Claims" of a Contributor + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the + License, by the making, using, selling, offering for sale, having + made, import, or transfer of either its Contributions or its + Contributor Version. + +1.12. "Secondary License" + means either the GNU General Public License, Version 2.0, the GNU + Lesser General Public License, Version 2.1, the GNU Affero General + Public License, Version 3.0, or any later versions of those + licenses. + +1.13. "Source Code Form" + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that + controls, is controlled by, or is under common control with You. For + purposes of this definition, "control" means (a) the power, direct + or indirect, to cause the direction or management of such entity, + whether by contract or otherwise, or (b) ownership of more than + fifty percent (50%) of the outstanding shares or beneficial + ownership of such entity. + +1.15. "Data" + means any data extracted from an electronic device with or without + use of Covered Software, and/or analysed using Covered Software or + a Larger Work. + +1.16. "Device Owner" (or "Device Owners") + means an individal or a legal entity with legal ownership of an + electronic device which is being analysed through the use of + Covered Software or a Larger Work, or from which Data was extracted + for subsequent analysis. + +1.17. "Data Owner" (or "Data Owners") + means an individial or group of individuals who made use of the + electronic device from which Data that is extracted and/or analyzed + originated. "Data Owner" might or might not differ from "Device + Owner". + +2. License Grants and Conditions +-------------------------------- + +2.1. Grants + +Each Contributor hereby grants You a world-wide, royalty-free, +non-exclusive license: + +(a) under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + +(b) under Patent Claims of such Contributor to make, use, sell, offer + for sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + +The licenses granted in Section 2.1 with respect to any Contribution +become effective for each Contribution on the date the Contributor first +distributes such Contribution. + +2.3. Limitations on Grant Scope + +The licenses granted in this Section 2 are the only rights granted under +this License. No additional rights or licenses will be implied from the +distribution or licensing of Covered Software under this License. +Notwithstanding Section 2.1(b) above, no patent license is granted by a +Contributor: + +(a) for any code that a Contributor has removed from Covered Software; + or + +(b) for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + +(c) under Patent Claims infringed by Covered Software in the absence of + its Contributions. + +This License does not grant any rights in the trademarks, service marks, +or logos of any Contributor (except as may be necessary to comply with +the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + +No Contributor makes additional grants as a result of Your choice to +distribute the Covered Software under a subsequent version of this +License (see Section 10.2) or under the terms of a Secondary License (if +permitted under the terms of Section 3.3). + +2.5. Representation + +Each Contributor represents that the Contributor believes its +Contributions are its original creation(s) or it has sufficient rights +to grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + +This License is not intended to limit any rights You have under +applicable copyright doctrines of fair use, fair dealing, or other +equivalents. + +2.7. Conditions + +Sections 3.0, 3.1, 3.2, 3.3, and 3.6 are conditions of the licenses +granted in Section 2.1. + +3. Responsibilities +------------------- + +3.0. Consensual Use Restriction + +Use of Covered Software or of a Larger Work is permitted provided that +the Data Owner must explicitly consent to the procedure, free from any +form of coercion, and must be fully informed about the nature of the +procedure, its privacy implications, and any data retention and disposal +policy. + +3.1. Distribution of Source Form + +All distribution of Covered Software in Source Code Form, including any +Modifications that You create or to which You contribute, must be under +the terms of this License. You must inform recipients that the Source +Code Form of the Covered Software is governed by the terms of this +License, and how they can obtain a copy of this License. You may not +attempt to alter or restrict the recipients' rights in the Source Code +Form. + +3.2. Distribution of Executable Form + +If You distribute Covered Software in Executable Form then: + +(a) such Covered Software must also be made available in Source Code + Form, as described in Section 3.1, and You must inform recipients of + the Executable Form how they can obtain a copy of such Source Code + Form by reasonable means in a timely manner, at a charge no more + than the cost of distribution to the recipient; and + +(b) You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter + the recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + +You may create and distribute a Larger Work under terms of Your choice, +provided that You also comply with the requirements of this License for +the Covered Software. If the Larger Work is a combination of Covered +Software with a work governed by one or more Secondary Licenses, and the +Covered Software is not Incompatible With Secondary Licenses, this +License permits You to additionally distribute such Covered Software +under the terms of such Secondary License(s), so that the recipient of +the Larger Work may, at their option, further distribute the Covered +Software under the terms of either this License or such Secondary +License(s). + +3.4. Notices + +You may not remove or alter the substance of any license notices +(including copyright notices, patent notices, disclaimers of warranty, +or limitations of liability) contained within the Source Code Form of +the Covered Software, except that You may alter any license notices to +the extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + +You may choose to offer, and to charge a fee for, warranty, support, +indemnity or liability obligations to one or more recipients of Covered +Software. However, You may do so only on Your own behalf, and not on +behalf of any Contributor. You must make it absolutely clear that any +such warranty, support, indemnity, or liability obligation is offered by +You alone, and You hereby agree to indemnify every Contributor for any +liability incurred by such Contributor as a result of warranty, support, +indemnity or liability terms You offer. You may include additional +disclaimers of warranty and limitations of liability specific to any +jurisdiction. + +4. Inability to Comply Due to Statute or Regulation +--------------------------------------------------- + +If it is impossible for You to comply with any of the terms of this +License with respect to some or all of the Covered Software due to +statute, judicial order, or regulation then You must: (a) comply with +the terms of this License to the maximum extent possible; and (b) +describe the limitations and the code they affect. Such description must +be placed in a text file included with all distributions of the Covered +Software under this License. Except to the extent prohibited by statute +or regulation, such description must be sufficiently detailed for a +recipient of ordinary skill to be able to understand it. + +5. Termination +-------------- + +5.1. The rights granted under this License will terminate automatically +if You fail to comply with any of its terms. However, if You become +compliant, then the rights granted under this License from a particular +Contributor are reinstated (a) provisionally, unless and until such +Contributor explicitly and finally terminates Your grants, and (b) on an +ongoing basis, if such Contributor fails to notify You of the +non-compliance by some reasonable means prior to 60 days after You have +come back into compliance. Moreover, Your grants from a particular +Contributor are reinstated on an ongoing basis if such Contributor +notifies You of the non-compliance by some reasonable means, this is the +first time You have received notice of non-compliance with this License +from such Contributor, and You become compliant prior to 30 days after +Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent +infringement claim (excluding declaratory judgment actions, +counter-claims, and cross-claims) alleging that a Contributor Version +directly or indirectly infringes any patent, then the rights granted to +You by any and all Contributors for the Covered Software under Section +2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all +end user license agreements (excluding distributors and resellers) which +have been validly granted by You or Your distributors under this License +prior to termination shall survive termination. + +************************************************************************ +* * +* 6. Disclaimer of Warranty * +* ------------------------- * +* * +* Covered Software is provided under this License on an "as is" * +* basis, without warranty of any kind, either expressed, implied, or * +* statutory, including, without limitation, warranties that the * +* Covered Software is free of defects, merchantable, fit for a * +* particular purpose or non-infringing. The entire risk as to the * +* quality and performance of the Covered Software is with You. * +* Should any Covered Software prove defective in any respect, You * +* (not any Contributor) assume the cost of any necessary servicing, * +* repair, or correction. This disclaimer of warranty constitutes an * +* essential part of this License. No use of any Covered Software is * +* authorized under this License except under this disclaimer. * +* * +************************************************************************ + +************************************************************************ +* * +* 7. Limitation of Liability * +* -------------------------- * +* * +* Under no circumstances and under no legal theory, whether tort * +* (including negligence), contract, or otherwise, shall any * +* Contributor, or anyone who distributes Covered Software as * +* permitted above, be liable to You for any direct, indirect, * +* special, incidental, or consequential damages of any character * +* including, without limitation, damages for lost profits, loss of * +* goodwill, work stoppage, computer failure or malfunction, or any * +* and all other commercial damages or losses, even if such party * +* shall have been informed of the possibility of such damages. This * +* limitation of liability shall not apply to liability for death or * +* personal injury resulting from such party's negligence to the * +* extent applicable law prohibits such limitation. Some * +* jurisdictions do not allow the exclusion or limitation of * +* incidental or consequential damages, so this exclusion and * +* limitation may not apply to You. * +* * +************************************************************************ + +8. Litigation +------------- + +Any litigation relating to this License may be brought only in the +courts of a jurisdiction where the defendant maintains its principal +place of business and such litigation shall be governed by laws of that +jurisdiction, without reference to its conflict-of-law provisions. +Nothing in this Section shall prevent a party's ability to bring +cross-claims or counter-claims. + +9. Miscellaneous +---------------- + +This License represents the complete agreement concerning the subject +matter hereof. If any provision of this License is held to be +unenforceable, such provision shall be reformed only to the extent +necessary to make it enforceable. Any law or regulation which provides +that the language of a contract shall be construed against the drafter +shall not be used to construe this License against a Contributor. + +10. Versions of the License +--------------------------- + +10.1. New Versions + +Claudio Guarnieri is the license steward. Except as provided in Section +10.3, no one other than the license steward has the right to modify or +publish new versions of this License. Each version will be given a +distinguishing version number. + +10.2. Effect of New Versions + +You may distribute the Covered Software under the terms of the version +of the License under which You originally received the Covered Software, +or under the terms of any subsequent version published by the license +steward. + +10.3. Modified Versions + +If you create software not governed by this License, and you want to +create a new license for such software, you may create and use a +modified version of this License if you rename the license and remove +any references to the name of the license steward (except to note that +such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary +Licenses + +If You choose to distribute Source Code Form that is Incompatible With +Secondary Licenses under the terms of this version of the License, the +notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice +------------------------------------------- + + This Source Code Form is subject to the terms of the MVT License, + v. 1.0. If a copy of the MVT License was not distributed with this + file, You can obtain one at TODO. + +If it is not possible or desirable to put the notice in a particular +file, then You may include the notice in a location (such as a LICENSE +file in a relevant directory) where a recipient would be likely to look +for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice +--------------------------------------------------------- + + This Source Code Form is "Incompatible With Secondary Licenses", as + defined by the MVT License, v. 1.0. + + +This license is an adaption of Mozilla Public License, v. 2.0. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..f20eee1 --- /dev/null +++ b/Makefile @@ -0,0 +1,10 @@ +PWD = $(shell pwd) + +clean: + rm -rf $(PWD)/build $(PWD)/dist $(PWD)/mvt.egg-info + +dist: + python3 setup.py sdist bdist_wheel + +upload: + python3 -m twine upload dist/* diff --git a/README.md b/README.md new file mode 100644 index 0000000..15e1293 --- /dev/null +++ b/README.md @@ -0,0 +1,11 @@ +

+ +

+ +# Mobile Verification Toolkit + + + +Mobile Verification Toolkit (MVT) is a collection of utilities to simplify and automate the process of gathering forensic traces helpful to identify a potential compromise of Android and iOS devices. + +[Please check out the documentation](https://mvt.readthedocs.io/). diff --git a/dev/mvt-android b/dev/mvt-android new file mode 100755 index 0000000..4ac6a2f --- /dev/null +++ b/dev/mvt-android @@ -0,0 +1,13 @@ +#!/usr/bin/env python3 +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021 MVT Project Developers. +# See the file 'LICENSE' for usage and copying permissions, or find a copy at +# https://github.com/mvt-project/mvt/blob/main/LICENSE + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from mvt import android +android.cli() diff --git a/dev/mvt-ios b/dev/mvt-ios new file mode 100755 index 0000000..ab89c1e --- /dev/null +++ b/dev/mvt-ios @@ -0,0 +1,13 @@ +#!/usr/bin/env python3 +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021 MVT Project Developers. +# See the file 'LICENSE' for usage and copying permissions, or find a copy at +# https://github.com/mvt-project/mvt/blob/main/LICENSE + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from mvt import ios +ios.cli() diff --git a/docs/android.md b/docs/android.md new file mode 100644 index 0000000..10ade98 --- /dev/null +++ b/docs/android.md @@ -0,0 +1,24 @@ +# Checking an Android Device + +In order to use `mvt-android` you need to connect your Android device to your computer. You will then need to [enable USB debugging](https://developer.android.com/studio/debug/dev-options#enable>) on the Android device. + +If this is the first time you connect to this device, you will need to approve the authentication keys through a prompt that will appear on your Android device. + +Now you can launch `mvt-android` and specify the `download-apks` command and the path to the folder where you want to store the extracted data: + +```bash +mvt-android download-apks --output /path/to/folder +``` + +Optionally, you can decide to enable lookups of the SHA256 hash of all the extracted APKs on [VirusTotal](https://www.virustotal.com) and/or [Koodous](https://www.koodous.com). While these lookups do not provide any conclusive assessment on all of the extracted APKs, they might highlight any known malicious ones: + +```bash +mvt-android download-apks --output /path/to/folder --virustotal +mvt-android download-apks --output /path/to/folder --koodous +``` + +Or, to launch all available lookups:: + +```bash +mvt-android download-apks --output /path/to/folder --all-checks +``` diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..3f39d3e --- /dev/null +++ b/docs/index.md @@ -0,0 +1,13 @@ +

+ +

+ +# Mobile Verification Toolkit + +Mobile Verification Toolkit (MVT) is a tool to facilitate the [consensual forensic analysis](introduction.md#consensual-forensics) of Android and iOS devices, for the purpose of identifying traces of compromise. + +In this documentation you will find instructions on how to install and run the `mvt-ios` and `mvt-android` commands, and guidance on how to interpret the extracted results. + +## Resources + +[:fontawesome-brands-python: Python Package](https://pypi.org/project/mvt){: .md-button .md-button--primary } [:fontawesome-brands-github: GitHub](https://github.com/mvt-project/mvt){: .md-button } diff --git a/docs/install.md b/docs/install.md new file mode 100644 index 0000000..072b828 --- /dev/null +++ b/docs/install.md @@ -0,0 +1,43 @@ +# Installation + +Before proceeding, please note that mvt requires Python 3.6+ to run. While it should be available on most operating systems, please make sure of that before proceeding. + +## Dependencies on Linux + +First install some basic dependencies that will be necessary to build all required tools: + +```bash +sudo apt install python3 python3-pip libusb-1.0-0 +``` + +*libusb-1.0-0* is not required if you intend to only use `mvt-ios` and not `mvt-android`. + +## Dependencies on Mac + +Running MVT on Mac requires Xcode and [homebrew](https://brew.sh) to be installed. + +In order to install dependencies use: + +```bash +brew install python3 libusb +``` + +*libusb* is not required if you intend to only use `mvt-ios` and not `mvt-android`. + +## Installing MVT + +If you haven't done so, you can add this to your `.bashrc` or `.zshrc` file in order to add locally installed Pypi binaries to your `$PATH`: + +```bash +export PATH=$PATH:~/.local/bin +``` + +Then you can install MVT directly: + +```bash +git clone https://github.com/mvt-project/mvt.git +cd mvt +pip3 install . +``` + +You now should have the `mvt-ios` and `mvt-android` utilities installed. diff --git a/docs/introduction.md b/docs/introduction.md new file mode 100644 index 0000000..06d66ea --- /dev/null +++ b/docs/introduction.md @@ -0,0 +1,17 @@ +# Introduction + +Mobile Verification Toolkit (MVT) is a collection of utilities designed to facilitate the consensual forensic acquisition of iOS and Android devices for the purpose of identifying any signs of compromise. MVT's capabilities are continuously evolving, but some of its key features include: + +- Decrypt encrypted iOS backups. +- Process and parse records from numerous iOS system and apps databases, logs and system analytics. +- Extract installed applications from Android devices. +- Extract diagnostic information from Android devices through the adb protocol. +- Compare extracted records to a provided list of malicious indicators in STIX2 format. +- Generate JSON logs of extracted records, and separate JSON logs of all detected malicious traces. +- Generate a unified chronological timeline of extracted records, along with a timeline all detected malicious traces. + +## Consensual Forensics + +While MVT is capable of extracting and processing various types of very personal records typically found on a mobile phone (such as calls history, SMS and WhatsApp messages, etc.), this is intended to help identify potential attack vectors such as malicious SMS messages leading to exploitation. + +MVT's purpose is not to facilitate adversial forensics of non-consenting individuals' devices. The use of MVT and derivative products to extract and/or analyse data originating from devices used by individuals not consenting to the procedure is explicitly prohibited in the [license](license.md). diff --git a/docs/ios/backup/check.md b/docs/ios/backup/check.md new file mode 100644 index 0000000..6313a7c --- /dev/null +++ b/docs/ios/backup/check.md @@ -0,0 +1,63 @@ +# Check a Backup with mvt-ios + +The backup might take some time. It is best to make sure the phone remains unlocked during the backup process. Afterwards, a new folder will be created under the path you specified using the UDID of the iPhone you backed up. + +## Decrypting a backup + +In case you have an encrypted backup, you will need to decrypt it first. This can be done with `mvt-ios` as well: + + $ mvt-ios decrypt-backup --help + Usage: mvt-ios decrypt-backup [OPTIONS] BACKUP_PATH + + Decrypt an encrypted iTunes backup + + Options: + -d, --destination TEXT Path to the folder where to store the decrypted + backup [required] + + -p, --password TEXT Password to use to decrypt the backup NOTE: This + argument is mutually exclusive with arguments: + [key_file]. + + -k, --key-file PATH File containing raw encryption key to use to decrypt + the backup NOTE: This argument is mutually exclusive + with arguments: [password]. + + --help Show this message and exit. + +You can specify either a password via command-line or pass a key file, and you need to specify a destination path where the decrypted backup will be stored. Following is an example usage of `decrypt-backup`: + +```bash +mvt-ios decrypt-backup -p password -d /path/to/decrypted /path/to/backup +``` + +## Run `mvt-ios` on a Backup + +Once you have a decrypted backup available for analysis you can use the `check-backup` subcommand: + + $ mvt-ios check-backup --help + Usage: mvt-ios check-backup [OPTIONS] BACKUP_PATH + + Extract artifacts from an iTunes backup + + Options: + -i, --iocs PATH Path to indicators file + -o, --output PATH Specify a path to a folder where you want to store JSON + results + + -f, --fast Avoid running time/resource consuming features + -l, --list-modules Print list of available modules and exit + -m, --module TEXT Name of a single module you would like to run instead of + all + + --help Show this message and exit. + +Following is a basic usage of `check-backup`: + +```bash +mvt-ios check-backup --output /path/to/output/ /path/to/backup/udid/ +``` + +This command will create a few JSON files containing the results from the extraction. If you do not specify a `--output` option, `mvt-ios` will just process the data without storing results on disk. + +Through the `--iocs` argument you can specify a [STIX2](https://oasis-open.github.io/cti-documentation/stix/intro) file defining a list of malicious indicators to check against the records extracted from the backup by mvt. Any matches will be highlighted in the terminal output as well as saved in the output folder using a "*_detected*" suffix to the JSON file name. diff --git a/docs/ios/backup/itunes.md b/docs/ios/backup/itunes.md new file mode 100644 index 0000000..985129b --- /dev/null +++ b/docs/ios/backup/itunes.md @@ -0,0 +1 @@ +# Backup with iTunes app diff --git a/docs/ios/backup/libimobiledevice.md b/docs/ios/backup/libimobiledevice.md new file mode 100644 index 0000000..892717c --- /dev/null +++ b/docs/ios/backup/libimobiledevice.md @@ -0,0 +1,15 @@ +# Backup with libimobiledevice + +If you have correctly [installed libimobiledevice](../install.md) you can easily generate an iTunes backup using the `idevicebackup2` tool included in the suite. First, you might want to ensure that backup encryption is enabled (**note: encrypted backup contain more data than unencrypted backups**): + +```bash +idevicebackup2 backup encryption on +``` + +Note that if a backup password was previously set on this device, you might need to use the same or change it. You can try changing password using `idevicebackup2 backup changepw` or resetting the password by resetting only the settings through the iPhone's Settings app. + +Once ready, you can proceed performing the backup: + +```bash +idevicebackup2 backup --full /path/to/backup/ +``` diff --git a/docs/ios/filesystem/check.md b/docs/ios/filesystem/check.md new file mode 100644 index 0000000..f1e9f3e --- /dev/null +++ b/docs/ios/filesystem/check.md @@ -0,0 +1,30 @@ +# Check a Filesystem Dump with `mvt-ios` + +When you are ready, you can proceed running `mvt-ios` against the filesystemp dump or mount point: + + $ mvt-ios check-fs --help + Usage: mvt-ios check-fs [OPTIONS] DUMP_PATH + + Extract artifacts from a full filesystem dump + + Options: + -i, --iocs PATH Path to indicators file + -o, --output PATH Specify a path to a folder where you want to store JSON + results + + -f, --fast Avoid running time/resource consuming features + -l, --list-modules Print list of available modules and exit + -m, --module TEXT Name of a single module you would like to run instead of + all + + --help Show this message and exit. + +Following is an example of basic usage of `check-fs`: + +```bash +mvt-ios check-fs /path/to/filesystem/dump/ --output /path/to/output/ +``` + +This command will create a few JSON files containing the results from the extraction. If you do not specify a `--output` option, `mvt-ios` will just process the data without storing results on disk. + +Through the `--iocs` argument you can specify a [STIX2](https://oasis-open.github.io/cti-documentation/stix/intro) file defining a list of malicious indicators to check against the records extracted from the backup by mvt. Any matches will be highlighted in the terminal output as well as saved in the output folder using a "*_detected*" suffix to the JSON file name. diff --git a/docs/ios/filesystem/dump.md b/docs/ios/filesystem/dump.md new file mode 100644 index 0000000..bdaf5a9 --- /dev/null +++ b/docs/ios/filesystem/dump.md @@ -0,0 +1,48 @@ +# Dumping the filesystem + +While iTunes backup provide a lot of very useful databases and diagnistic data, in some cases you might want to jailbreak the device and perform a full filesystem dump. In that case, you should take a look at [checkra1n](https://checkra.in/), which provides an easy way to obtain root on most recent iPhone models. + +!!! warning + Before you checkra1n any device, make sure you take a full backup, and that you are prepared to do a full factory reset before restoring it. Even after using checkra1n's "Restore System", some traces of the jailbreak are still left on the device and [apps with anti-jailbreaks will be able to detect them](https://github.com/checkra1n/BugTracker/issues/279) and stop functioning. + +After having jailbroken the device, you should be able to access the phone over ssh. In order to do this you will typically need to use iproxy, which on Debian/Ubuntu systems can be installed with `libusbmuxd-tools`. Run the command: + +```bash +iproxy 2222 44 +``` + +Now you will be able to ssh as root to localhost on port 2222 and password `alpine`. Note: if you used a jailbreak other than checkra1n, you might need to specify a different port number instead of 44. + +At this point you need to get access to the content of the device from your computer. One way is to run a command like `ssh root@localhost -p 2222 tar czf - /private > dump.tar.gz` which will save a tarball on the host of the */private/* folder from the phone. This will take a while. + +Alternatively, you can try run `sftp-server` for iOS and mount the filesystem locally using `sshfs`. + + +## Use `sshfs` on iOS + +If you decide to try to use sshfs, you first have to download locally a compiled copy of sftp-server: + +```bash +wget https://github.com/dweinstein/openssh-ios/releases/download/v7.5/sftp-server +``` + +Then upload the binary to the iPhone: + +```bash +scp -P2222 sftp-server root@localhost:. +``` + +You will need to ssh into the device and set some entitlements in order to allow `sftp-server` to run. This entitlements can be copied from an existing binary: + +```bash +chmod +x sftp-server +ldid -e /binpack/bin/sh > /tmp/sh-ents +ldid -S /tmp/sh-ents sftp-server +``` + +Now you can create a folder on the host and use it as a mount point (**note:** do not create this folder in /tmp/): + +```bash +mkdir root_mount +sshfs -p 2222 -o sftp_server=/var/root/sftp-server root@localhost:/ root_mount +``` diff --git a/docs/ios/install.md b/docs/ios/install.md new file mode 100644 index 0000000..7f8e57a --- /dev/null +++ b/docs/ios/install.md @@ -0,0 +1,55 @@ +# Install libimobiledevice + +Before proceeding with doing any acquisition of iOS devices we recommend installing [libimobiledevice](https://www.libimobiledevice.org/) utilities. These utilities will become useful when extracting crash logs and generating iTunes backups. Because the utilities and its libraries are subject to frequent changes in response to new versions of iOS, you might want to consider compiling libimobiledevice utilities from sources. Otherwise, if available, you can try installing packages available in your distribution: + +```bash +sudo apt install libimobiledevice-utils +``` + +On Mac, you can try installing it from brew: + +```bash +brew install --HEAD libimobiledevice +``` + +If you have a reasonably recent version of libimobiledevice in your package manager, it might work straight out of the box. Try connecting your iOS device to your computer via USB and run: + +```bash +ideviceinfo +``` + +If you encounter unexpected issues, uninstall the packages and try compiling libimobiledevcice from sources. + +## Compile libimobiledevice from sources + +!!! warning + The following instructions are a best effort. The installation from source requires several steps, and it is likely some have been forgotten here and that won't work for you. You will likely need to fiddle around a bit before getting this right. + +Make sure you have uninstalled all the libimobiledevice tools from your package manage: + +```bash +sudo apt remove --purge libimobiledevice-utils libimobiledevice-dev libimobiledevice6 libplist-dev libplist3 libusbmuxd-dev libusbmuxd-tools libusbmuxd4 libusbmuxd6 usbmuxd +``` + +Firstly you need to install [libplist](https://github.com/libimobiledevice/libplist). Then you can install [libusbmuxd](https://github.com/libimobiledevice/libusbmuxd). + +Now you should be able to to download and install the actual suite of tools at [https://github.com/libimobiledevice/libimobiledevice](https://github.com/libimobiledevice/libimobiledevice). + +You can now also build and install [usbmuxd](https://github.com/libimobiledevice/usbmuxd). + +## Making sure everything works fine. + +Once the idevice tools are available you can check if everything works fine by connecting your iOS device and running: + +```bash +ideviceinfo +``` + +This should some many details on the connected iOS device. If you are connecting the device to your laptop for the first time, it will require to unlock and enter the PIN code on the mobile device. If it complains that no device is connected and the mobile device is indeed plugged in through the USB cable, you might need to do this first, although typically the pairing is automatically done when connecting the device: + +```bash +sudo usbmuxd -f -d +idevicepair pair +``` + +Again, it will ask to unlock the phone and enter the PIN code. diff --git a/docs/ios/methodology.md b/docs/ios/methodology.md new file mode 100644 index 0000000..bbf15d8 --- /dev/null +++ b/docs/ios/methodology.md @@ -0,0 +1,15 @@ +# iOS Forensic Methodology + +Before jumping into acquiring and analyzing data from an iOS device, you should evaluate what is your precise plan of action. Because multiple options are available to you, you should define and familiarize with the most effective forensic methodology in each case. + +#### Filesystem Dump + +You will need to decide whether to attempt to jailbreak the device and obtain a full filesystem dump, or not. + +While access the full filesystem allows to extact data that would otherwise be unavailable, it might not always be possible to jailbreak a certain iPhone model or version of iOS. In addition, depending on the type of jailbreak available, doing so might compromise some important records, pollute others, or potentially cause unintended malfunctioning of the device later in case it is used again. + +If you are not expected to return the phone, you might want to consider to attempt a jailbreak after having exhausted all other options, including a backup. + +#### iTunes Backup + +An alternative option is to generate an iTunes backup (in most recent version of mac OS, they are no longer launched from iTunes, but directly from Finder). While backups only provide a subset of the files stored on the device, in many cases it might be sufficient to at least detect some suspicious artifacts. Backups encrypted with a password will have some additional interesting records not available in unencrypted ones, such as Safari history, Safari state, etc. diff --git a/docs/ios/records.md b/docs/ios/records.md new file mode 100644 index 0000000..f5dad16 --- /dev/null +++ b/docs/ios/records.md @@ -0,0 +1,278 @@ +# Records extracted by `mvt-ios` + +In this page you can find a (reasonably) up-to-date breakdown of the files created by MVT when performing an analysis of logs, backups or filesystem dumps. + +## Records extracted by `check-fs` or `check-backup` + +### `cache_files.json` + +!!! info "Availability" + Backup: :material-close: + Full filesystem dump: :material-check: + +This JSON file is created by mvt-ios' `CacheFiles` module. The module extracts records from all SQLite database files stored on disk with the name *Cache.db*. These databases typically contain data from iOS' [internal URL caching](https://developer.apple.com/documentation/foundation/nsurlcache). Through this module you might be able to recover records of HTTP requests and responses performed my applications as well as system services, that would otherwise be unavailable. For example, you might see HTTP requests part of an exploitation chain performed by an iOS service attempting to download a first stage malicious payload. + +If indicators are provided through the command-line, they are checked against the requested URL. Any matches are stored in *cache_files_detected.json*. + +--- + +### `calls.json` + +!!! info "Availability" + Backup: :material-check: + Full filesystem dump: :material-check: + +This JSON file is created by mvt-ios' `Calls` module. The module extracts records from a SQLite database located at */private/var/mobile/Library/CallHistoryDB/CallHistory.storedata*, which contains records of incoming and outgoing calls, including from messaging apps such as WhatsApp or Skype. + +--- + +### `chrome_favicon.json` + +!!! info "Availability" + Backup: :material-check: + Full filesystem dump: :material-check: + +This JSON file is created by mvt-ios' `ChromeFavicon` module. The module extracts records from a SQLite database located at */private/var/mobile/Containers/Data/Application/\*/Library/Application Support/Google/Chrome/Default/Favicons*, which contains a mapping of favicons' URLs and the visited URLs which loaded them. + +If indicators are provided through the command-line, they are checked against both the favicon URL and the visited URL. Any matches are stored in *chrome_favicon_detected.json*. + +--- + +### `chrome_history.json` + +!!! info "Availability" + Backup: :material-check: + Full filesystem dump: :material-check: + +This JSON file is created by mvt-ios' `ChromeHistory` module. The module extracts records from a SQLite database located at */private/var/mobile/Containers/Data/Application/\*/Library/Application Support/Google/Chrome/Default/History*, which contains a history of URL visits. + +If indicators a provided through the command-line, they are checked against the visited URL. Any matches are stored in *chrome_history_detected.json*. + +--- + +### `contacts.json` + +!!! info "Availability" + Backup: :material-check: + Full filesystem dump: :material-check: + +This JSON file is created by mvt-ios' `Contacts` module. The module extracts records from a SQLite database located at */private/var/mobile/Library/AddressBook/AddressBook.sqlitedb*, which contains records from the phone's address book. While this database obviously would not contain any malicious indicators per se, you might want to use it to compare records from other apps (such as iMessage, SMS, etc.) to filter those originating from unknown origins. + +--- + +### `firefox_favicon.json` + +!!! info "Availability" + Backup: :material-check: + Full filesystem dump: :material-check: + +This JSON file is created by mvt-ios' `FirefoxFavicon` module. The module extracts records from a SQLite database located at */private/var/mobile/profile.profile/browser.db*, which contains a mapping of favicons' URLs and the visited URLs which loaded them. + +If indicators are provided through the command-line, they are checked against both the favicon URL and the visited URL. Any matches are stored in *firefox_favicon_detected.json*. + +--- + +### `firefox_history.json` + +!!! info "Availability" + Backup: :material-check: + Full filesystem dump: :material-check: + +This JSON file is created by mvt-ios' `FirefoxHistory` module. The module extracts records from a SQLite database located at */private/var/mobile/profile.profile/browser.db*, which contains a history of URL visits. + +If indicators a provided through the command-line, they are checked against the visited URL. Any matches are stored in *firefox_history_detected.json*. + +--- + +### `id_status_cache.json` + +!!! info "Availability" + Backup: :material-check: + Full filesystem dump: :material-check: + +This JSON file is created by mvt-ios' `IDStatusCache` module. The module extracts records from a plist file located at */private/var/mobile/Library/Preferences/com.apple.identityservices.idstatuscache.plist*, which contains a cache of Apple user ID authentication. This chance will indicate when apps like Facetime and iMessage first established contacts with other registered Apple IDs. This is significant because it might contain traces of malicious accounts involved in exploitation of those apps. + +--- + +### `interaction_c.json` + +!!! info "Availability" + Backup: :material-check: + Full filesystem dump: :material-check: + +This JSON file is created by mvt-ios' `InteractionC` module. The module extracts records from a SQLite database located at */private/var/mobile/Library/CoreDuet/People/interactionC.db*, which contains details about user interactions with installed apps. + +--- + +### `locationd_clients.json` + +!!! info "Availability" + Backup: :material-check: + Full filesystem dump: :material-check: + +This JSON file is created by mvt-ios' `LocationdClients` module. The module extracts records from a plist file located at */private/var/mobile/Library/Caches/locationd/clients.plist*, which contains a cache of apps which requested access to location services. + +--- + +### `manifest.json` + +!!! info "Availability" + Backup: :material-check: + Full filesystem dump: :material-close: + +This JSON file is created by mvt-ios' `Manifest` module. The module extracts records from the SQLite database *Manifest.db* contained in iTunes backups, and which indexes the locally backed-up files to the original paths on the iOS device. + +If indicators are provided through the command-line, they are checked against the original relative path in case. In some cases, there might be records of files created containing a domain name in their name, for example in the case of browser cache folders. Any matches are stored in *manifest_detected.json*. + +--- + +### `datausage.json` + +!!! info "Availability" + Backup: :material-check: + Full filesystem dump: :material-check: + +This JSON file is created by mvt-ios' `Datausage` module. The module extracts records from a SQLite database located */private/var/wireless/Library/Databases/DataUsage.sqlite*, which contains a history of data usage by processes running on the system. Besides the network statistics, these records are particularly important because they might show traces of malicious process executions and the relevant timeframe. In particular, processes which do not have a valid bundle ID might require particular attention. + +If indicators are provided through the command-line, they are checked against the process names. Any matches are stored in *datausage_detected.json*. If running on a full filesystem dump and if the `--fast` flag was not enabled by command-line, mvt-ios will highlight processes which look suspicious and check the presence of a binary file of the same name in the dump. + +--- + +### `netusage.json` + +!!! info "Availability" + Backup: :material-close: + Full filesystem dump: :material-check: + +This JSON file is created by mvt-ios' `Netusage` module. The module extracts records from a SQLite database located */private/var/networkd/netusage.sqlite*, which contains a history of data usage by processes running on the system. Besides the network statistics, these records are particularly important because they might show traces of malicious process executions and the relevant timeframe. In particular, processes which do not have a valid bundle ID might require particular attention. + +If indicators are provided through the command-line, they are checked against the process names. Any matches are stored in *netusage_detected.json*. If running on a full filesystem dump and if the `--fast` flag was not enabled by command-line, mvt-ios will highlight processes which look suspicious and check the presence of a binary file of the same name in the dump. + +--- + +### `safari_browser_state.json` + +!!! info "Availability" + Backup: :material-check: + Full filesystem dump: :material-check: + +This JSON file is created by mvt-ios' `SafariBrowserState` module. The module extracts records from the SQLite databases located at */private/var/mobile/Library/Safari/BrowserState.db* or */private/var/mobile/Containers/Data/Application/\*/Library/Safari/BrowserState.db*, which contain records of opened tabs. + +If indicators a provided through the command-line, they are checked against the visited URL. Any matches are stored in *safari_browser_state_detected.json*. + +--- + +### `safari_favicon.json` + +!!! info "Availability" + Backup: :material-close: + Full filesystem dump: :material-check: + +This JSON file is created by mvt-ios' `SafariFavicon` module. The module extracts records from the SQLite databases located at */private/var/mobile/Library/Image Cache/Favicons/Favicons.db* or */private/var/mobile/Containers/Data/Application/\*/Library/Image Cache/Favicons/Favicons.db*, which contain mappings of favicons' URLs and the visited URLs which loaded them. + +If indicators are provided through the command-line, they are checked against both the favicon URL and the visited URL. Any matches are stored in *safari_favicon_detected.json*. + +--- + +### `safari_history.json` + +!!! info "Availability" + Backup: :material-check: + Full filesystem dump: :material-check: + +This JSON file is created by mvt-ios' `SafariHistory` module. The module extracts records from the SQLite databases located at */private/var/mobile/Library/Safari/History.db* or */private/var/mobile/Containers/Data/Application/\*/Library/Safari/History.db*, which contain a history of URL visits. + +If indicators are provided through the command-line, they are checked against the visited URL. Any matches are stored in *safari_history_detected.json*. + +--- + +### `sms.json` + +!!! info "Availability" + Backup: :material-check: + Full filesystem dump: :material-check: + +This JSON file is created by mvt-ios' `SMS` module. The module extracts a list of SMS messages containing HTTP links from the SQLite database located at */private/var/mobile/Library/SMS/sms.db*. + +If indicators are provided through the command-line, they are checked against the extracted HTTP links. Any matches are stored in *sms_detected.json*. + +--- + +### `sms_attachments.json` + +!!! info "Availability" + Backup: :material-check: + Full filesystem dump: :material-check: + +This JSON file is created by mvt-ios' `SMSAttachments` module. The module extracts details about attachments sent via SMS or iMessage from the same database used by the `SMS` module. These records might be useful to indicate unique patterns that might be indicative of exploitation attempts leveraging potential vulnerabilities in file format parsers or other forms of file handling by the Messages app. + +--- + +### `version_history.json` + +!!! info "Availability" + Backup: :material-close: + Full filesystem dump: :material-check: + +This JSON file is created by mvt-ios' `IOSVersionHistory` module. The module extracts records of iOS software updates from analytics plist files located at */private/var/db/analyticsd/Analytics-Journal-\*.ips*. + +--- + +### `webkit_indexeddb.json` + +!!! info "Availability" + Backup: :material-close: + Full filesystem dump: :material-check: + +This JSON file is created by mvt-ios' `WebkitIndexedDB` module. The module extracts a list of file and folder names located at the following path */private/var/mobile/Containers/Data/Application/\*/Library/WebKit/WebsiteData/IndexedDB*, which contains IndexedDB files created by any app installed on the device. + +If indicators are provided through the command-line, they are checked against the extracted names. Any matches are stored in *webkit_indexeddb_detected.json*. + +--- + +### `webkit_local_storage.json` + +!!! info "Availability" + Backup: :material-close: + Full filesystem dump: :material-check: + +This JSON file is created by mvt-ios' `WebkitLocalStorage` module. The module extracts a lsit of file and folder names located at the following path */private/var/mobile/Containers/Data/Application/\*/Library/WebKit/WebsiteData/LocalStorage/*, which contains local storage files created by any app installed on the device. + +If indicators are provided through the command-line, they are checked against the extracted names. Any matches are stored in *webkit_local_storage_detected.json*. + +--- + +### `webkit_safari_view_service.json` + +!!! info "Availability" + Backup: :material-close: + Full filesystem dump: :material-check: + +This JSON file is created by mvt-ios' `WebkitSafariViewService` module. The module extracts a list of file and folder names located at the following path */private/var/mobile/Containers/Data/Application/\*/SystemData/com.apple.SafariViewService/Library/WebKit/WebsiteData/*, which contains files cached by SafariVewService. + +If indicators are provided through the command-line, they are checked against the extracted names. Any matches are stored in *webkit_safari_view_service_detected.json*. + +--- + +### `webkit_session_resource_log.json` + +!!! info "Availability" + Backup: :material-check: + Full filesystem dump: :material-check: + +This JSON file is created by mvt-ios' `WebkitSessionResourceLog` module. The module extracts records from plist files with the name *full_browsing_session_resourceLog.plist*, which contain records of resources loaded by different domains visited. + +If indicators are provided through the command-line, they are checked against the extract domains. Any matches are stored in *webkit_session_resource_log_detected.json*. + +--- + +### `whatsapp.json` + +!!! info "Availability" + Backup: :material-check: + Full filesystem dump: :material-check: + +This JSON file is created by mvt-ios' `WhatsApp` module. The module extracts a list of WhatsApp messages containing HTTP links from the SQLite database located at *private/var/mobile/Containers/Shared/AppGroup/\*/ChatStorage.sqlite*. + +If indicators are provided through the command-line, they are checked against the extracted HTTP links. Any matches are stored in *whatsapp_detected.json*. + diff --git a/docs/license.md b/docs/license.md new file mode 100644 index 0000000..e824e49 --- /dev/null +++ b/docs/license.md @@ -0,0 +1,9 @@ +# MVT License + +The purpose of MVT is to facilitate the ***consensual forensic analysis*** of devices of those who might be targets of sophisticated mobile spyware attacks, especially members of civil society and marginalized communities. We do not want MVT to enable privacy violations of non-consenting individuals. Therefore, the goal of this license is to prohibit the use of MVT (and any other software licensed the same) for the purpose of *adversarial forensics*. + +In order to achieve this, MVT is released under an adaptation of [Mozilla Public License v2.0](https://www.mozilla.org/MPL). This modified license includes a new clause 3.0, "Consensual Use Restriction" which permits the use of the licensed software (and any *"Larger Work"* derived from it) exclusively with the explicit consent of the person/s whose data is being extracted and/or analysed (*"Data Owner"*). + +**Please note:** because this license imposes some use restrictions, software using it infringes *"freedom 0"* of Free Software Foundation's [*"Free Software Definition"*](https://www.gnu.org/philosophy/free-sw.en.html), and therefore can not be considered "Free Software" according to FSF. Similarly, it might infringe the *"No Discrimination Against Fields of Endeavor"* criteria in Open Source Initiative's [*"Open Source Definition"*](https://opensource.org/osd), therefore software using this license might also not be considered "Open Source" according to OSI. + +[Read the LICENSE](https://github.com/mvt-project/mvt/blob/main/LICENSE) diff --git a/docs/mvt.png b/docs/mvt.png new file mode 100644 index 0000000..c469f4e Binary files /dev/null and b/docs/mvt.png differ diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 0000000..d12d77e --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,43 @@ +site_name: Mobile Verification Toolkit +repo_url: https://github.com/mvt-project/mvt +edit_uri: edit/main/docs/ +copyright: Copyright © 2021 MVT Project Developers +site_description: Mobile Verification Toolkit Documentation +markdown_extensions: + - attr_list + - admonition + - pymdownx.emoji: + emoji_index: !!python/name:materialx.emoji.twemoji + emoji_generator: !!python/name:materialx.emoji.to_svg + - pymdownx.superfences + - pymdownx.inlinehilite + - pymdownx.highlight: + use_pygments: true +theme: + name: material + features: + - tabs +plugins: + - search + - mkdocstrings +extra: + social: + - icon: fontawesome/brands/github + link: https://github.com/mvt-project/mvt +nav: + - Welcome: "index.md" + - Introduction: "introduction.md" + - Installation: "install.md" + - MVT for iOS: + - iOS Forensic Methodology: "ios/methodology.md" + - Install libimobiledevice: "ios/install.md" + - Check an iTunes Backup: + - Backup with iTunes app: "ios/backup/itunes.md" + - Backup with libimobiledevice: "ios/backup/libimobiledevice.md" + - Check a Backup with mvt-ios: "ios/backup/check.md" + - Check a Filesystem Dump: + - Dumping the filesystem: "ios/filesystem/dump.md" + - Check a Filesystem Dump with mvt-ios: "ios/filesystem/check.md" + - Records extracted by mvt-ios: "ios/records.md" + - MVT for Android: "android.md" + - License: "license.md" diff --git a/mvt/__init__.py b/mvt/__init__.py new file mode 100644 index 0000000..2512a63 --- /dev/null +++ b/mvt/__init__.py @@ -0,0 +1,4 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021 MVT Project Developers. +# See the file 'LICENSE' for usage and copying permissions, or find a copy at +# https://github.com/mvt-project/mvt/blob/main/LICENSE diff --git a/mvt/android/__init__.py b/mvt/android/__init__.py new file mode 100644 index 0000000..2ae7e01 --- /dev/null +++ b/mvt/android/__init__.py @@ -0,0 +1,6 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021 MVT Project Developers. +# See the file 'LICENSE' for usage and copying permissions, or find a copy at +# https://github.com/mvt-project/mvt/blob/main/LICENSE + +from .cli import cli diff --git a/mvt/android/cli.py b/mvt/android/cli.py new file mode 100644 index 0000000..eccf9e1 --- /dev/null +++ b/mvt/android/cli.py @@ -0,0 +1,156 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021 MVT Project Developers. +# See the file 'LICENSE' for usage and copying permissions, or find a copy at +# https://github.com/mvt-project/mvt/blob/main/LICENSE + +import os +import sys +import click +import argparse +import logging +from rich.logging import RichHandler + +from mvt.common.module import run_module, save_timeline +from mvt.common.indicators import Indicators +from .download_apks import DownloadAPKs +from .lookups.koodous import koodous_lookup +from .lookups.virustotal import virustotal_lookup +from .modules.adb import ADB_MODULES +from .modules.backup import BACKUP_MODULES + +# Setup logging using Rich. +LOG_FORMAT = "[%(name)s] %(message)s" +logging.basicConfig(level="INFO", format=LOG_FORMAT, handlers=[ + RichHandler(show_path=False, log_time_format="%X")]) +log = logging.getLogger(__name__) + +# Help messages of repeating options. +OUTPUT_HELP_MESSAGE = "Specify a path to a folder where you want to store JSON results" + + +#============================================================================== +# Main +#============================================================================== +@click.group(invoke_without_command=False) +def cli(): + return + + +#============================================================================== +# Download APKs +#============================================================================== +@cli.command("download-apks", help="Download all or non-safelisted installed APKs installed on the device") +@click.option("--all-apks", "-a", is_flag=True, + help="Extract all packages installed on the phone, even those marked as safe") +@click.option("--virustotal", "-v", is_flag=True, help="Check packages on VirusTotal") +@click.option("--koodous", "-k", is_flag=True, help="Check packages on Koodous") +@click.option("--all-checks", "-A", is_flag=True, help="Run all available checks") +@click.option("--output", "-o", type=click.Path(exists=True), + help="Specify a path to a folder where you want to store JSON results") +@click.option("--from-file", "-f", type=click.Path(exists=True), + help="Instead of acquiring from phone, load an existing packages.json file for lookups (mainly for debug purposes)") +def download_apks(all_apks, virustotal, koodous, all_checks, output, from_file): + try: + if from_file: + download = DownloadAPKs.from_json(from_file) + else: + if not output: + log.critical("You need to specify an output folder (with --output, -o) when extracting APKs from a device") + sys.exit(-1) + + download = DownloadAPKs(output_folder=output, all_apks=all_apks) + download.run() + + packages = download.packages + + if len(packages) == 0: + return + + if virustotal or all_checks: + virustotal_lookup(packages) + + if koodous or all_checks: + koodous_lookup(packages) + except KeyboardInterrupt: + print("") + sys.exit(-1) + + +#============================================================================== +# Checks through ADB +#============================================================================== +@cli.command("check-adb", help="Check an Android device over adb") +@click.option("--iocs", "-i", type=click.Path(exists=True), help="Path to indicators file") +@click.option("--output", "-o", type=click.Path(exists=True), + help="Specify a path to a folder where you want to store JSON results") +@click.option("--list-modules", "-l", is_flag=True, help="Print list of available modules and exit") +@click.option("--module", "-m", help="Name of a single module you would like to run instead of all") +def check_adb(iocs, output, list_modules, module): + if list_modules: + log.info("Following is the list of available check-adb modules:") + for adb_module in ADB_MODULES: + log.info(" - %s", adb_module.__name__) + + return + + log.info("Checking Android through adb bridge") + + if iocs: + # Pre-load indicators for performance reasons. + log.info("Loading indicators from provided file at %s", iocs) + indicators = Indicators(iocs) + + timeline = [] + timeline_detected = [] + for adb_module in ADB_MODULES: + if module and adb_module.__name__ != module: + continue + + m = adb_module(output_folder=output, log=logging.getLogger(adb_module.__module__)) + + if iocs: + indicators.log = m.log + m.indicators = indicators + + run_module(m) + timeline.extend(m.timeline) + timeline_detected.extend(m.timeline_detected) + + if output: + if len(timeline) > 0: + save_timeline(timeline, os.path.join(output, "timeline.csv")) + if len(timeline_detected) > 0: + save_timeline(timeline_detected, os.path.join(output, "timeline_detected.csv")) + +#============================================================================== +# Check ADB backup +#============================================================================== +@cli.command("check-backup", help="Check an Android Backup") +@click.option("--iocs", "-i", type=click.Path(exists=True), help="Path to indicators file") +@click.option("--output", "-o", type=click.Path(exists=True), help=OUTPUT_HELP_MESSAGE) +@click.argument("BACKUP_PATH", type=click.Path(exists=True)) +def check_backup(iocs, output, backup_path): + log.info("Checking ADB backup located at: %s", backup_path) + + if iocs: + # Pre-load indicators for performance reasons. + log.info("Loading indicators from provided file at %s", iocs) + indicators = Indicators(iocs) + + if os.path.isfile(backup_path): + log.critical("The path you specified is a not a folder!") + + if os.path.basename(backup_path) == "backup.ab": + log.info("You can use ABE (https://github.com/nelenkov/android-backup-extractor) " \ + "to extract 'backup.ab' files!") + sys.exit(-1) + + for module in BACKUP_MODULES: + m = module(base_folder=backup_path, output_folder=output, + log=logging.getLogger(module.__module__)) + + if iocs: + indicators.log = m.log + m.indicators = indicators + + run_module(m) diff --git a/mvt/android/data/root_binaries.txt b/mvt/android/data/root_binaries.txt new file mode 100644 index 0000000..5dc0ffa --- /dev/null +++ b/mvt/android/data/root_binaries.txt @@ -0,0 +1,10 @@ +su +busybox +supersu +Superuser.apk +KingoUser.apk +SuperSu.apk +magisk +magiskhide +magiskinit +magiskpolicy diff --git a/mvt/android/data/root_packages.txt b/mvt/android/data/root_packages.txt new file mode 100644 index 0000000..7011e07 --- /dev/null +++ b/mvt/android/data/root_packages.txt @@ -0,0 +1,25 @@ +com.noshufou.android.su +com.noshufou.android.su.elite +eu.chainfire.supersu +com.koushikdutta.superuser +com.thirdparty.superuser +com.yellowes.su +com.koushikdutta.rommanager +com.koushikdutta.rommanager.license +com.dimonvideo.luckypatcher +com.chelpus.lackypatch +com.ramdroid.appquarantine +com.ramdroid.appquarantinepro +com.devadvance.rootcloak +com.devadvance.rootcloakplus +de.robv.android.xposed.installer +com.saurik.substrate +com.zachspong.temprootremovejb +com.amphoras.hidemyroot +com.amphoras.hidemyrootadfree +com.formyhm.hiderootPremium +com.formyhm.hideroot +me.phh.superuser +eu.chainfire.supersu.pro +com.kingouser.com +com.topjohnwu.magisk diff --git a/mvt/android/data/safe_packages.txt b/mvt/android/data/safe_packages.txt new file mode 100644 index 0000000..2898237 --- /dev/null +++ b/mvt/android/data/safe_packages.txt @@ -0,0 +1,182 @@ +android +android.auto_generated_rro__ +android.autoinstalls.config.google.nexus +com.android.backupconfirm +com.android.bips +com.android.bluetooth +com.android.bluetoothmidiservice +com.android.bookmarkprovider +com.android.calllogbackup +com.android.captiveportallogin +com.android.carrierconfig +com.android.carrierdefaultapp +com.android.cellbroadcastreceiver +com.android.certinstaller +com.android.chrome +com.android.companiondevicemanager +com.android.connectivity.metrics +com.android.cts.ctsshim +com.android.cts.priv.ctsshim +com.android.defcontainer +com.android.documentsui +com.android.dreams.basic +com.android.egg +com.android.emergency +com.android.externalstorage +com.android.facelock +com.android.hotwordenrollment +com.android.hotwordenrollment.okgoogle +com.android.hotwordenrollment.tgoogle +com.android.hotwordenrollment.xgoogle +com.android.htmlviewer +com.android.inputdevices +com.android.keychain +com.android.location.fused +com.android.managedprovisioning +com.android.mms.service +com.android.mtp +com.android.musicfx +com.android.nfc +com.android.omadm.service +com.android.pacprocessor +com.android.phone +com.android.printspooler +com.android.providers.blockednumber +com.android.providers.calendar +com.android.providers.contacts +com.android.providers.downloads +com.android.providers.downloads.ui +com.android.providers.media +com.android.providers.partnerbookmarks +com.android.providers.settings +com.android.providers.telephony +com.android.providers.userdictionary +com.android.proxyhandler +com.android.retaildemo +com.android.safetyregulatoryinfo +com.android.sdm.plugins.connmo +com.android.sdm.plugins.dcmo +com.android.sdm.plugins.diagmon +com.android.sdm.plugins.sprintdm +com.android.server.telecom +com.android.service.ims +com.android.service.ims.presence +com.android.settings +com.android.sharedstoragebackup +com.android.shell +com.android.statementservice +com.android.stk +com.android.systemui +com.android.systemui.theme.dark +com.android.vending +com.android.vpndialogs +com.android.vzwomatrigger +com.android.wallpaperbackup +com.android.wallpaper.livepicker +com.breel.wallpapers +com.customermobile.preload.vzw +com.google.android.apps.cloudprint +com.google.android.apps.docs +com.google.android.apps.docs.editors.docs +com.google.android.apps.enterprise.dmagent +com.google.android.apps.gcs +com.google.android.apps.helprtc +com.google.android.apps.inputmethod.hindi +com.google.android.apps.maps +com.google.android.apps.messaging +com.google.android.apps.nexuslauncher +com.google.android.apps.photos +com.google.android.apps.pixelmigrate +com.google.android.apps.tachyon +com.google.android.apps.turbo +com.google.android.apps.tycho +com.google.android.apps.wallpaper +com.google.android.apps.wallpaper.nexus +com.google.android.apps.work.oobconfig +com.google.android.apps.youtube.vr +com.google.android.asdiv +com.google.android.backuptransport +com.google.android.calculator +com.google.android.calendar +com.google.android.carrier +com.google.android.carrier.authdialog +com.google.android.carrierentitlement +com.google.android.carriersetup +com.google.android.configupdater +com.google.android.contacts +com.google.android.deskclock +com.google.android.dialer +com.google.android.euicc +com.google.android.ext.services +com.google.android.ext.shared +com.google.android.feedback +com.google.android.gm +com.google.android.gms +com.google.android.gms.policy_auth +com.google.android.gms.policy_sidecar_o +com.google.android.gms.setup +com.google.android.GoogleCamera +com.google.android.googlequicksearchbox +com.google.android.gsf +com.google.android.gsf.login +com.google.android.hardwareinfo +com.google.android.hiddenmenu +com.google.android.ims +com.google.android.inputmethod.japanese +com.google.android.inputmethod.korean +com.google.android.inputmethod.latin +com.google.android.inputmethod.pinyin +com.google.android.instantapps.supervisor +com.google.android.keep +com.google.android.marvin.talkback +com.google.android.music +com.google.android.nexusicons +com.google.android.onetimeinitializer +com.google.android.packageinstaller +com.google.android.partnersetup +com.google.android.printservice.recommendation +com.google.android.setupwizard +com.google.android.soundpicker +com.google.android.storagemanager +com.google.android.syncadapters.contacts +com.google.android.tag +com.google.android.talk +com.google.android.tetheringentitlement +com.google.android.theme.pixel +com.google.android.tts +com.google.android.videos +com.google.android.vr.home +com.google.android.vr.inputmethod +com.google.android.webview +com.google.android.wfcactivation +com.google.android.youtube +com.google.ar.core +com.google.intelligence.sense +com.google.modemservice +com.google.pixel.wahoo.gfxdrv +com.google.SSRestartDetector +com.google.tango +com.google.vr.apps.ornament +com.google.vr.vrcore +com.htc.omadm.trigger +com.qti.qualcomm.datastatusnotification +com.qualcomm.atfwd +com.qualcomm.embms +com.qualcomm.fastdormancy +com.qualcomm.ltebc_vzw +com.qualcomm.qcrilmsgtunnel +com.qualcomm.qti.ims +com.qualcomm.qti.networksetting +com.qualcomm.qti.telephonyservice +com.qualcomm.qti.uceShimService +com.qualcomm.shutdownlistner +com.qualcomm.timeservice +com.qualcomm.vzw_api +com.quicinc.cne.CNEService +com.verizon.llkagent +com.verizon.mips.services +com.verizon.obdm +com.verizon.obdm_permissions +com.verizon.services +com.vzw.apnlib +qualcomm.com.vzw_msdc_api diff --git a/mvt/android/download_apks.py b/mvt/android/download_apks.py new file mode 100644 index 0000000..5dd4cbc --- /dev/null +++ b/mvt/android/download_apks.py @@ -0,0 +1,212 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021 MVT Project Developers. +# See the file 'LICENSE' for usage and copying permissions, or find a copy at +# https://github.com/mvt-project/mvt/blob/main/LICENSE + +import os +import json +import logging +import pkg_resources +from tqdm import tqdm + +from mvt.common.utils import get_sha256_from_file_path +from .modules.adb.base import AndroidExtraction + +log = logging.getLogger(__name__) + +# TODO: Would be better to replace tqdm with rich.progress to reduce +# the number of dependencies. Need to investigate whether +# it's possible to have a simialr callback system. +class PullProgress(tqdm): + """PullProgress is a tqdm update system for APK downloads.""" + + def update_to(self, file_name, current, total): + if total is not None: + self.total = total + self.update(current - self.n) + + +class Package: + """Package indicates a package name and all the files associated with it.""" + + def __init__(self, name, files=None): + self.name = name + self.files = files or [] + + +class DownloadAPKs(AndroidExtraction): + """DownloadAPKs is the main class operating the download of APKs + from the device.""" + + def __init__(self, output_folder=None, all_apks=False, packages=None): + """Initialize module. + :param output_folder: Path to the folder where data should be stored + :param all_apks: Boolean indicating whether to download all packages + or filter known-goods + :param packages: Provided list of packages, typically for JSON checks + """ + super().__init__(file_path=None, base_folder=None, + output_folder=output_folder) + + self.output_folder_apk = None + self.packages = packages or [] + self.all_apks = all_apks + + self._safe_packages = [] + + @classmethod + def from_json(cls, json_path): + """Initialize this class from an existing packages.json file. + :param json_path: Path to the packages.json file to parse. + """ + with open(json_path, "r") as handle: + data = json.load(handle) + + packages = [] + for entry in data: + package = Package(entry["name"], entry["files"]) + packages.append(package) + + return cls(packages=packages) + + def _load_safe_packages(self): + """Load known-good package names. + """ + safe_packages_path = os.path.join("data", "safe_packages.txt") + safe_packages_string = pkg_resources.resource_string(__name__, safe_packages_path) + safe_packages_list = safe_packages_string.decode("utf-8").split("\n") + self._safe_packages.extend(safe_packages_list) + + def _clean_output(self, output): + """Clean adb shell command output. + :param output: Command output to clean. + """ + return output.strip().replace("package:", "") + + def get_packages(self): + """Retrieve package names from the device using adb. + """ + log.info("Retrieving package names ...") + + if not self.all_apks: + self._load_safe_packages() + + output = self._adb_command("pm list packages") + total = 0 + for line in output.split("\n"): + package_name = self._clean_output(line) + if package_name == "": + continue + + total += 1 + + if not self.all_apks and package_name in self._safe_packages: + continue + + if package_name not in self.packages: + self.packages.append(Package(package_name)) + + log.info("There are %d packages installed on the device. I selected %d for inspection.", + total, len(self.packages)) + + def pull_package_file(self, package_name, remote_path): + """Pull files related to specific package from the device. + :param package_name: Name of the package to download + :param remote_path: Path to the file to download + :returns: Path to the local copy + """ + log.info("Downloading %s ...", remote_path) + + file_name = "" + if "==/" in remote_path: + file_name = "_" + remote_path.split("==/")[1].replace(".apk", "") + + local_path = os.path.join(self.output_folder_apk, + f"{package_name}{file_name}.apk") + name_counter = 0 + while True: + if not os.path.exists(local_path): + break + + name_counter += 1 + local_path = os.path.join(self.output_folder_apk, + f"{package_name}{file_name}_{name_counter}.apk") + + try: + with PullProgress(unit='B', unit_divisor=1024, unit_scale=True, + miniters=1) as pp: + self._adb_download(remote_path, local_path, + progress_callback=pp.update_to) + except Exception as e: + log.exception("Failed to pull package file from %s: %s", + remote_path, e) + self._adb_reconnect() + return None + + return local_path + + def pull_packages(self): + """Download all files of all selected packages from the device. + """ + log.info("Starting extraction of installed APKs at folder %s", self.output_folder) + + if not os.path.exists(self.output_folder): + os.mkdir(self.output_folder) + + log.info("Downloading packages from device. This might take some time ...") + + self.output_folder_apk = os.path.join(self.output_folder, "apks") + if not os.path.exists(self.output_folder_apk): + os.mkdir(self.output_folder_apk) + + total_packages = len(self.packages) + counter = 0 + for package in self.packages: + counter += 1 + + log.info("[%d/%d] Package: %s", counter, total_packages, package.name) + + try: + output = self._adb_command(f"pm path {package.name}") + output = self._clean_output(output) + if not output: + continue + except Exception as e: + log.exception("Failed to get path of package %s: %s", package.name, e) + self._adb_reconnect() + continue + + # Sometimes the package path contains multiple lines for multiple apks. + # We loop through each line and download each file. + for path in output.split("\n"): + device_path = path.strip() + file_path = self.pull_package_file(package.name, device_path) + if not file_path: + continue + + # We add the apk metadata to the package object. + package.files.append({ + "path": device_path, + "local_name": file_path, + "sha256": get_sha256_from_file_path(file_path), + }) + + def save_json(self): + """Save the results to the package.json file. + """ + json_path = os.path.join(self.output_folder, "packages.json") + packages = [] + for package in self.packages: + packages.append(package.__dict__) + + with open(json_path, "w") as handle: + json.dump(packages, handle, indent=4) + + def run(self): + """Run all steps of fetch-apk. + """ + self._adb_connect() + self.get_packages() + self.pull_packages() + self.save_json() + self._adb_disconnect() diff --git a/mvt/android/lookups/__init__.py b/mvt/android/lookups/__init__.py new file mode 100644 index 0000000..2512a63 --- /dev/null +++ b/mvt/android/lookups/__init__.py @@ -0,0 +1,4 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021 MVT Project Developers. +# See the file 'LICENSE' for usage and copying permissions, or find a copy at +# https://github.com/mvt-project/mvt/blob/main/LICENSE diff --git a/mvt/android/lookups/koodous.py b/mvt/android/lookups/koodous.py new file mode 100644 index 0000000..78f5c0f --- /dev/null +++ b/mvt/android/lookups/koodous.py @@ -0,0 +1,57 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021 MVT Project Developers. +# See the file 'LICENSE' for usage and copying permissions, or find a copy at +# https://github.com/mvt-project/mvt/blob/main/LICENSE + +import requests +import logging + +from rich.text import Text +from rich.table import Table +from rich.progress import track +from rich.console import Console + +log = logging.getLogger(__name__) + +def koodous_lookup(packages): + log.info("Looking up all extracted files on Koodous (www.koodous.com)") + log.info("This might take a while...") + + table = Table(title="Koodous Packages Detections") + table.add_column("Package name") + table.add_column("File name") + table.add_column("Trusted") + table.add_column("Detected") + table.add_column("Rating") + + total_packages = len(packages) + for i in track(range(total_packages), description=f"Looking up {total_packages} packages..."): + package = packages[i] + for file in package.files: + url = f"https://api.koodous.com/apks/{file['sha256']}" + res = requests.get(url) + report = res.json() + + row = [package.name, file["local_name"]] + + if "package_name" in report: + trusted = "no" + if report["trusted"]: + trusted = Text("yes", "green bold") + + detected = "no" + if report["detected"]: + detected = Text("yes", "red bold") + + rating = "0" + if int(report["rating"]) < 0: + rating = Text(str(report["rating"]), "red bold") + + row.extend([trusted, detected, rating]) + else: + row.extend(["n/a", "n/a", "n/a"]) + + table.add_row(*row) + + console = Console() + console.print(table) diff --git a/mvt/android/lookups/virustotal.py b/mvt/android/lookups/virustotal.py new file mode 100644 index 0000000..eaac727 --- /dev/null +++ b/mvt/android/lookups/virustotal.py @@ -0,0 +1,92 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021 MVT Project Developers. +# See the file 'LICENSE' for usage and copying permissions, or find a copy at +# https://github.com/mvt-project/mvt/blob/main/LICENSE + +import requests +import logging +from rich.text import Text +from rich.table import Table +from rich.progress import track +from rich.console import Console + +log = logging.getLogger(__name__) + +def get_virustotal_report(hashes): + apikey = "233f22e200ca5822bd91103043ccac138b910db79f29af5616a9afe8b6f215ad" + url = f"https://www.virustotal.com/partners/sysinternals/file-reports?apikey={apikey}" + + items = [] + for sha256 in hashes: + items.append({ + "autostart_location": "", + "autostart_entry": "", + "hash": sha256, + "local_name": "", + "creation_datetime": "", + }) + headers = {"User-Agent": "VirusTotal", "Content-Type": "application/json"} + res = requests.post(url, headers=headers, json=items) + + if res.status_code == 200: + report = res.json() + return report["data"] + else: + log.error("Unexpected response from VirusTotal: %s", res.status_code) + return None + +def virustotal_lookup(packages): + log.info("Looking up all extracted files on VirusTotal (www.virustotal.com)") + + unique_hashes = [] + for package in packages: + for file in package.files: + if file["sha256"] not in unique_hashes: + unique_hashes.append(file["sha256"]) + + total_unique_hashes = len(unique_hashes) + + detections = {} + def virustotal_query(batch): + report = get_virustotal_report(batch) + if not report: + return + + for entry in report: + if entry["hash"] not in detections and entry["found"] is True: + detections[entry["hash"]] = entry["detection_ratio"] + + batch = [] + for i in track(range(total_unique_hashes), description=f"Looking up {total_unique_hashes} files..."): + file_hash = unique_hashes[i] + batch.append(file_hash) + if len(batch) == 25: + virustotal_query(batch) + batch = [] + + if batch: + virustotal_query(batch) + + table = Table(title="VirusTotal Packages Detections") + table.add_column("Package name") + table.add_column("File path") + table.add_column("Detections") + + for package in packages: + for file in package.files: + row = [package.name, file["local_name"]] + + if file["sha256"] in detections: + detection = detections[file["sha256"]] + positives = detection.split("/")[0] + if int(positives) > 0: + row.append(Text(detection, "red bold")) + else: + row.append(detection) + else: + row.append("not found") + + table.add_row(*row) + + console = Console() + console.print(table) diff --git a/mvt/android/modules/__init__.py b/mvt/android/modules/__init__.py new file mode 100644 index 0000000..2512a63 --- /dev/null +++ b/mvt/android/modules/__init__.py @@ -0,0 +1,4 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021 MVT Project Developers. +# See the file 'LICENSE' for usage and copying permissions, or find a copy at +# https://github.com/mvt-project/mvt/blob/main/LICENSE diff --git a/mvt/android/modules/adb/__init__.py b/mvt/android/modules/adb/__init__.py new file mode 100644 index 0000000..ddca6b4 --- /dev/null +++ b/mvt/android/modules/adb/__init__.py @@ -0,0 +1,18 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021 MVT Project Developers. +# See the file 'LICENSE' for usage and copying permissions, or find a copy at +# https://github.com/mvt-project/mvt/blob/main/LICENSE + +from .chrome_history import ChromeHistory +from .dumpsys_batterystats import DumpsysBatterystats +from .dumpsys_packages import DumpsysPackages +from .dumpsys_procstats import DumpsysProcstats +from .processes import Processes +from .sms import SMS +from .whatsapp import Whatsapp +from .packages import Packages +from .rootbinaries import RootBinaries + +ADB_MODULES = [ChromeHistory, SMS, Whatsapp, Processes, + DumpsysBatterystats, DumpsysProcstats, + DumpsysPackages, Packages, RootBinaries] diff --git a/mvt/android/modules/adb/base.py b/mvt/android/modules/adb/base.py new file mode 100644 index 0000000..31aa931 --- /dev/null +++ b/mvt/android/modules/adb/base.py @@ -0,0 +1,166 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021 MVT Project Developers. +# See the file 'LICENSE' for usage and copying permissions, or find a copy at +# https://github.com/mvt-project/mvt/blob/main/LICENSE + +import os +import sys +import time +import logging +import tempfile +from adb_shell.adb_device import AdbDeviceUsb +from adb_shell.auth.keygen import keygen, write_public_keyfile +from adb_shell.auth.sign_pythonrsa import PythonRSASigner +from adb_shell.exceptions import DeviceAuthError, AdbCommandFailureException +from usb1 import USBErrorBusy, USBErrorAccess + +from mvt.common.module import MVTModule + +log = logging.getLogger(__name__) + +ADB_KEY_PATH = os.path.expanduser("~/.android/adbkey") +ADB_PUB_KEY_PATH = os.path.expanduser("~/.android/adbkey.pub") + +class AndroidExtraction(MVTModule): + """This class provides a base for all Android extraction modules.""" + + def __init__(self, file_path=None, base_folder=None, output_folder=None, + fast_mode=False, log=None, results=[]): + """Initialize Android extraction module. + :param file_path: Path to the database file to parse + :param base_folder: Path to a base folder containing an Android dump + :param output_folder: Path to the folder where to store extraction + results + """ + super().__init__(file_path=file_path, base_folder=base_folder, + output_folder=output_folder, fast_mode=fast_mode, + log=log, results=results) + + self.device = None + + def _adb_check_keys(self): + """Make sure Android adb keys exist. + """ + if not os.path.exists(ADB_KEY_PATH): + keygen(ADB_KEY_PATH) + + if not os.path.exists(ADB_PUB_KEY_PATH): + write_public_keyfile(ADB_KEY_PATH, ADB_PUB_KEY_PATH) + + def _adb_connect(self): + """Connect to the device over adb. + """ + self._adb_check_keys() + + with open(ADB_KEY_PATH, "rb") as handle: + priv_key = handle.read() + + signer = PythonRSASigner("", priv_key) + self.device = AdbDeviceUsb() + + while True: + try: + self.device.connect(rsa_keys=[signer], auth_timeout_s=5) + except (USBErrorBusy, USBErrorAccess): + log.critical("Device is busy, maybe run `adb kill-server` and try again.") + sys.exit(-1) + except DeviceAuthError: + log.error("You need to authorize this computer on the Android device. Retrying in 5 seconds...") + time.sleep(5) + except Exception as e: + log.critical(e) + sys.exit(-1) + else: + break + + def _adb_disconnect(self): + """Close adb connection to the device. + """ + self.device.close() + + def _adb_reconnect(self): + """Reconnect to device using adb. + """ + log.info("Reconnecting ...") + self._adb_disconnect() + self._adb_connect() + + def _adb_command(self, command): + """Execute an adb shell command. + :param command: Shell command to execute + :returns: Output of command + """ + return self.device.shell(command) + + def _adb_check_if_root(self): + """Check if we have a `su` binary on the Android device. + :returns: Boolean indicating whether a `su` binary is present or not + """ + return bool(self._adb_command("[ ! -f /sbin/su ] || echo 1")) + + def _adb_root_or_die(self): + """Check if we have a `su` binary, otherwise raise an Exception. + """ + if not self._adb_check_if_root(): + raise Exception("The Android device does not seem to have a `su` binary. Cannot run this module.") + + def _adb_command_as_root(self, command): + """Execute an adb shell command. + :param command: Shell command to execute as root + :returns: Output of command + """ + return self._adb_command(f"su -c {command}") + + def _adb_download(self, remote_path, local_path, progress_callback=None): + """Download a file form the device. + :param remote_path: Path to download from the device + :param local_path: Path to where to locally store the copy of the file + :param progress_callback: Callback for download progress bar + """ + try: + self.device.pull(remote_path, local_path, progress_callback) + except AdbCommandFailureException as e: + raise Exception(f"Unable to download file {remote_path}: {e}") + + def _adb_process_file(self, remote_path, process_routine): + """Download a local copy of a file which is only accessible as root. + This is a wrapper around process_routine. + :param remote_path: Path of the file on the device to process + :param process_routine: Function to be called on the local copy of the + downloaded file + """ + # Connect to the device over adb. + self._adb_connect() + # Check if we have root, if not raise an Exception. + self._adb_root_or_die() + + # We create a temporary local file. + tmp = tempfile.NamedTemporaryFile() + local_path = tmp.name + local_name = os.path.basename(tmp.name) + new_remote_path = f"/sdcard/Download/{local_name}" + + # We copy the file from the data folder to /sdcard/. + cp = self._adb_command_as_root(f"cp {remote_path} {new_remote_path}") + if cp.startswith("cp: ") and "No such file or directory" in cp: + raise Exception(f"Unable to process file {remote_path}: File not found") + elif cp.startswith("cp: ") and "Permission denied" in cp: + raise Exception(f"Unable to process file {remote_path}: Permission denied") + + # We download from /sdcard/ to the local temporary file. + self._adb_download(new_remote_path, local_path) + + # Launch the provided process routine! + process_routine(local_path) + + # Delete the local copy. + tmp.close() + # Delete the copy on /sdcard/. + self._adb_command(f"rm -f {new_remote_path}") + # Disconnect from the device. + self._adb_disconnect() + + def run(self): + """Run the main procedure. + """ + raise NotImplementedError diff --git a/mvt/android/modules/adb/chrome_history.py b/mvt/android/modules/adb/chrome_history.py new file mode 100644 index 0000000..6b223d2 --- /dev/null +++ b/mvt/android/modules/adb/chrome_history.py @@ -0,0 +1,70 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021 MVT Project Developers. +# See the file 'LICENSE' for usage and copying permissions, or find a copy at +# https://github.com/mvt-project/mvt/blob/main/LICENSE + +import os +import sqlite3 +import logging + +from mvt.common.utils import convert_chrometime_to_unix, convert_timestamp_to_iso + +from .base import AndroidExtraction + +log = logging.getLogger(__name__) + +CHROME_HISTORY_PATH = "data/data/com.android.chrome/app_chrome/Default/History" + +class ChromeHistory(AndroidExtraction): + """This module extracts records from Android's Chrome browsing history.""" + + def __init__(self, file_path=None, base_folder=None, output_folder=None, + fast_mode=False, log=None, results=[]): + super().__init__(file_path=file_path, base_folder=base_folder, + output_folder=output_folder, fast_mode=fast_mode, + log=log, results=results) + + def serialize(self, record): + return { + "timestamp": record["isodate"], + "module": self.__class__.__name__, + "event": "visit", + "data": f"{record['id']} - {record['url']} (visit ID: {record['visit_id']}, redirect source: {record['redirect_source']})" + } + + def _parse_db(self, db_path): + """Parse a Chrome History database file. + :param db_path: Path to the History database to process. + """ + conn = sqlite3.connect(db_path) + cur = conn.cursor() + cur.execute(""" + SELECT + urls.id, + urls.url, + visits.id, + visits.visit_time, + visits.from_visit + FROM urls + JOIN visits ON visits.url = urls.id + ORDER BY visits.visit_time; + """) + + for item in cur: + self.results.append(dict( + id=item[0], + url=item[1], + visit_id=item[2], + timestamp=item[3], + isodate=convert_timestamp_to_iso(convert_chrometime_to_unix(item[3])), + redirect_source=item[4], + )) + + cur.close() + conn.close() + + log.info("Extracted a total of %d history items", len(self.results)) + + def run(self): + self._adb_process_file(os.path.join("/", CHROME_HISTORY_PATH), + self._parse_db) diff --git a/mvt/android/modules/adb/dumpsys_batterystats.py b/mvt/android/modules/adb/dumpsys_batterystats.py new file mode 100644 index 0000000..ad90027 --- /dev/null +++ b/mvt/android/modules/adb/dumpsys_batterystats.py @@ -0,0 +1,45 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021 MVT Project Developers. +# See the file 'LICENSE' for usage and copying permissions, or find a copy at +# https://github.com/mvt-project/mvt/blob/main/LICENSE + +import os +import logging + +from .base import AndroidExtraction + +log = logging.getLogger(__name__) + +class DumpsysBatterystats(AndroidExtraction): + """This module extracts stats on battery consumption by processes.""" + + def __init__(self, file_path=None, base_folder=None, output_folder=None, + fast_mode=False, log=None, results=[]): + super().__init__(file_path=file_path, base_folder=base_folder, + output_folder=output_folder, fast_mode=fast_mode, + log=log, results=results) + + def run(self): + self._adb_connect() + + stats = self._adb_command("dumpsys batterystats") + if self.output_folder: + stats_path = os.path.join(self.output_folder, + "dumpsys_batterystats.txt") + with open(stats_path, "w") as handle: + handle.write(stats) + + log.info("Records from dumpsys batterystats stored at %s", + stats_path) + + history = self._adb_command("dumpsys batterystats --history") + if self.output_folder: + history_path = os.path.join(self.output_folder, + "dumpsys_batterystats_history.txt") + with open(history_path, "w") as handle: + handle.write(history) + + log.info("History records from dumpsys batterystats stored at %s", + history_path) + + self._adb_disconnect() diff --git a/mvt/android/modules/adb/dumpsys_packages.py b/mvt/android/modules/adb/dumpsys_packages.py new file mode 100644 index 0000000..45fbf68 --- /dev/null +++ b/mvt/android/modules/adb/dumpsys_packages.py @@ -0,0 +1,35 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021 MVT Project Developers. +# See the file 'LICENSE' for usage and copying permissions, or find a copy at +# https://github.com/mvt-project/mvt/blob/main/LICENSE + +import os +import logging + +from .base import AndroidExtraction + +log = logging.getLogger(__name__) + +class DumpsysPackages(AndroidExtraction): + """This module extracts stats on installed packages.""" + + def __init__(self, file_path=None, base_folder=None, output_folder=None, + fast_mode=False, log=None, results=[]): + super().__init__(file_path=file_path, base_folder=base_folder, + output_folder=output_folder, fast_mode=fast_mode, + log=log, results=results) + + def run(self): + self._adb_connect() + + output = self._adb_command("dumpsys package") + if self.output_folder: + packages_path = os.path.join(self.output_folder, + "dumpsys_packages.txt") + with open(packages_path, "w") as handle: + handle.write(output) + + log.info("Records from dumpsys package stored at %s", + packages_path) + + self._adb_disconnect() diff --git a/mvt/android/modules/adb/dumpsys_procstats.py b/mvt/android/modules/adb/dumpsys_procstats.py new file mode 100644 index 0000000..416f01f --- /dev/null +++ b/mvt/android/modules/adb/dumpsys_procstats.py @@ -0,0 +1,35 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021 MVT Project Developers. +# See the file 'LICENSE' for usage and copying permissions, or find a copy at +# https://github.com/mvt-project/mvt/blob/main/LICENSE + +import os +import logging + +from .base import AndroidExtraction + +log = logging.getLogger(__name__) + +class DumpsysProcstats(AndroidExtraction): + """This module extracts stats on memory consumption by processes.""" + + def __init__(self, file_path=None, base_folder=None, output_folder=None, + fast_mode=False, log=None, results=[]): + super().__init__(file_path=file_path, base_folder=base_folder, + output_folder=output_folder, fast_mode=fast_mode, + log=log, results=results) + + def run(self): + self._adb_connect() + + output = self._adb_command("dumpsys procstats") + if self.output_folder: + procstats_path = os.path.join(self.output_folder, + "dumpsys_procstats.txt") + with open(procstats_path, "w") as handle: + handle.write(output) + + log.info("Records from dumpsys procstats stored at %s", + procstats_path) + + self._adb_disconnect() diff --git a/mvt/android/modules/adb/packages.py b/mvt/android/modules/adb/packages.py new file mode 100644 index 0000000..dcdc375 --- /dev/null +++ b/mvt/android/modules/adb/packages.py @@ -0,0 +1,111 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021 MVT Project Developers. +# See the file 'LICENSE' for usage and copying permissions, or find a copy at +# https://github.com/mvt-project/mvt/blob/main/LICENSE + +import os +import logging +import pkg_resources + +from .base import AndroidExtraction + +log = logging.getLogger(__name__) + +class Packages(AndroidExtraction): + """This module extracts the list of installed packages.""" + + def __init__(self, file_path=None, base_folder=None, output_folder=None, + fast_mode=False, log=None, results=[]): + super().__init__(file_path=file_path, base_folder=base_folder, + output_folder=output_folder, fast_mode=fast_mode, + log=log, results=results) + + def serialize(self, record): + records = [] + + timestamps = [ + {"event": "package_install", "timestamp": record["timestamp"]}, + {"event": "package_first_install", "timestamp": record["first_install_time"]}, + {"event": "package_last_update", "timestamp": record["last_update_time"]}, + ] + + for ts in timestamps: + records.append({ + "timestamp": ts["timestamp"], + "module": self.__class__.__name__, + "event": ts["event"], + "data": f"{record['package_name']} (system: {record['system']}, third party: {record['third_party']})", + }) + + return records + + def check_indicators(self): + root_packages_path = os.path.join("..", "..", "data", "root_packages.txt") + root_packages_string = pkg_resources.resource_string(__name__, root_packages_path) + root_packages = root_packages_string.decode("utf-8").split("\n") + + for root_package in root_packages: + root_package = root_package.strip() + if not root_package: + continue + + if root_package in self.results: + self.log.warning("Found an installed package related to rooting/jailbreaking: \"%s\"", + root_package) + self.detected.append(root_package) + + def run(self): + self._adb_connect() + + packages = self._adb_command("pm list packages -U -u -i -f") + for line in packages.split("\n"): + line = line.strip() + if not line.startswith("package:"): + continue + + fields = line.split() + file_name, package_name = fields[0].split(":")[1].rsplit("=", 1) + installer = fields[1].split("=")[1].strip() + if installer == "null": + installer = None + uid = fields[2].split(":")[1].strip() + + dumpsys = self._adb_command(f"dumpsys package {package_name} | grep -A2 timeStamp").split("\n") + timestamp = dumpsys[0].split("=")[1].strip() + first_install = dumpsys[1].split("=")[1].strip() + last_update = dumpsys[2].split("=")[1].strip() + + self.results.append(dict( + package_name=package_name, + file_name=file_name, + installer=installer, + timestamp=timestamp, + first_install_time=first_install, + last_update_time=last_update, + uid=uid, + disabled=False, + system=False, + third_party=False, + )) + + cmds = [ + {"field": "disabled", "arg": "-d"}, + {"field": "system", "arg": "-s"}, + {"field": "third_party", "arg": "-3"}, + ] + for cmd in cmds: + output = self._adb_command(f"pm list packages {cmd['arg']}") + for line in output.split("\n"): + line = line.strip() + if not line.startswith("package:"): + continue + + package_name = line.split(":", 1)[1] + for i, result in enumerate(self.results): + if result["package_name"] == package_name: + self.results[i][cmd["field"]] = True + + self.log.info("Extracted at total of %d installed package names", + len(self.results)) + + self._adb_disconnect() diff --git a/mvt/android/modules/adb/processes.py b/mvt/android/modules/adb/processes.py new file mode 100644 index 0000000..0be051b --- /dev/null +++ b/mvt/android/modules/adb/processes.py @@ -0,0 +1,54 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021 MVT Project Developers. +# See the file 'LICENSE' for usage and copying permissions, or find a copy at +# https://github.com/mvt-project/mvt/blob/main/LICENSE + +import logging + +from .base import AndroidExtraction + +log = logging.getLogger(__name__) + +class Processes(AndroidExtraction): + """This module extracts details on running processes.""" + + def __init__(self, file_path=None, base_folder=None, output_folder=None, + fast_mode=False, log=None, results=[]): + super().__init__(file_path=file_path, base_folder=base_folder, + output_folder=output_folder, fast_mode=fast_mode, + log=log, results=results) + + def run(self): + self._adb_connect() + + output = self._adb_command("ps") + + for line in output.split("\n")[1:]: + line = line.strip() + if line == "": + continue + + fields = line.split() + proc = dict( + user=fields[0], + pid=fields[1], + parent_pid=fields[2], + vsize=fields[3], + rss=fields[4], + ) + + # Sometimes WCHAN is empty, so we need to re-align output fields. + if len(fields) == 8: + proc["wchan"] = "" + proc["pc"] = fields[5] + proc["name"] = fields[7] + elif len(fields) == 9: + proc["wchan"] = fields[5] + proc["pc"] = fields[6] + proc["name"] = fields[8] + + self.results.append(proc) + + self._adb_disconnect() + + log.info("Extracted records on a total of %d processes", len(self.results)) diff --git a/mvt/android/modules/adb/rootbinaries.py b/mvt/android/modules/adb/rootbinaries.py new file mode 100644 index 0000000..b591603 --- /dev/null +++ b/mvt/android/modules/adb/rootbinaries.py @@ -0,0 +1,47 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021 MVT Project Developers. +# See the file 'LICENSE' for usage and copying permissions, or find a copy at +# https://github.com/mvt-project/mvt/blob/main/LICENSE + +import os +import logging +import pkg_resources + +from .base import AndroidExtraction + +log = logging.getLogger(__name__) + +class RootBinaries(AndroidExtraction): + """This module extracts the list of installed packages.""" + + def __init__(self, file_path=None, base_folder=None, output_folder=None, + fast_mode=False, log=None, results=[]): + super().__init__(file_path=file_path, base_folder=base_folder, + output_folder=output_folder, fast_mode=fast_mode, + log=log, results=results) + + def run(self): + root_binaries_path = os.path.join("..", "..", "data", "root_binaries.txt") + root_binaries_string = pkg_resources.resource_string(__name__, root_binaries_path) + root_binaries = root_binaries_string.decode("utf-8").split("\n") + + self._adb_connect() + + for root_binary in root_binaries: + root_binary = root_binary.strip() + if not root_binary: + continue + + output = self._adb_command(f"which -a {root_binary}") + output = output.strip() + + if not output: + continue + + if "which: not found" in output: + continue + + self.detected.append(root_binary) + self.log.warning("Found root binary \"%s\"", root_binary) + + self._adb_disconnect() diff --git a/mvt/android/modules/adb/sms.py b/mvt/android/modules/adb/sms.py new file mode 100644 index 0000000..dcaccac --- /dev/null +++ b/mvt/android/modules/adb/sms.py @@ -0,0 +1,91 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021 MVT Project Developers. +# See the file 'LICENSE' for usage and copying permissions, or find a copy at +# https://github.com/mvt-project/mvt/blob/main/LICENSE + +import os +import sqlite3 +import logging + +from .base import AndroidExtraction +from mvt.common.utils import convert_timestamp_to_iso, check_for_links + +log = logging.getLogger(__name__) + +SMS_PATH = "data/data/com.google.android.apps.messaging/databases/bugle_db" + +class SMS(AndroidExtraction): + """This module extracts all SMS messages containing links.""" + + def __init__(self, file_path=None, base_folder=None, output_folder=None, + fast_mode=False, log=None, results=[]): + super().__init__(file_path=file_path, base_folder=base_folder, + output_folder=output_folder, fast_mode=fast_mode, + log=log, results=results) + + def serialize(self, record): + text = record["text"].replace("\n", "\\n") + return { + "timestamp": record["isodate"], + "module": self.__class__.__name__, + "event": f"sms_{record['direction']}", + "data": f"{record['number']}: \"{text}\"" + } + + def check_indicators(self): + if not self.indicators: + return + + for message in self.results: + if not "text" in message: + continue + + message_links = check_for_links(message["text"]) + if self.indicators.check_domains(message_links): + self.detected.append(message) + + def _parse_db(self, db_path): + """Parse an Android bugle_db SMS database file. + :param db_path: Path to the Android SMS database file to process + """ + conn = sqlite3.connect(db_path) + cur = conn.cursor() + cur.execute(""" + SELECT + ppl.normalized_destination AS number, + p.timestamp AS timestamp, + CASE WHEN m.sender_id IN + (SELECT _id FROM participants WHERE contact_id=-1) + THEN 2 ELSE 1 END incoming, p.text AS text + FROM messages m, conversations c, parts p, + participants ppl, conversation_participants cp + WHERE (m.conversation_id = c._id) + AND (m._id = p.message_id) + AND (cp.conversation_id = c._id) + AND (cp.participant_id = ppl._id); + """) + names = [description[0] for description in cur.description] + + for item in cur: + message = dict() + for index, value in enumerate(item): + message[names[index]] = value + + message["direction"] = ("received" if message["incoming"] == 1 else "sent") + message["isodate"] = convert_timestamp_to_iso(message["timestamp"]) + + # If we find links in the messages or if they are empty we add + # them to the list of results. + if check_for_links(message["text"]) or message["text"].strip() == "": + self.results.append(message) + + cur.close() + conn.close() + + log.info("Extracted a total of %d SMS messages containing links", len(self.results)) + + def run(self): + try: + self._adb_process_file(os.path.join("/", SMS_PATH), self._parse_db) + except Exception as e: + self.log.error(e) diff --git a/mvt/android/modules/adb/whatsapp.py b/mvt/android/modules/adb/whatsapp.py new file mode 100644 index 0000000..6d369d6 --- /dev/null +++ b/mvt/android/modules/adb/whatsapp.py @@ -0,0 +1,84 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021 MVT Project Developers. +# See the file 'LICENSE' for usage and copying permissions, or find a copy at +# https://github.com/mvt-project/mvt/blob/main/LICENSE + +import os +import sqlite3 +import logging + +from .base import AndroidExtraction +from mvt.common.utils import convert_timestamp_to_iso, check_for_links + +log = logging.getLogger(__name__) + +WHATSAPP_PATH = "data/data/com.whatsapp/databases/msgstore.db" + +class Whatsapp(AndroidExtraction): + """This module extracts all WhatsApp messages containing links.""" + + def __init__(self, file_path=None, base_folder=None, output_folder=None, + fast_mode=False, log=None, results=[]): + super().__init__(file_path=file_path, base_folder=base_folder, + output_folder=output_folder, fast_mode=fast_mode, + log=log, results=results) + + def serialize(self, record): + text = record["data"].replace("\n", "\\n") + return { + "timestamp": record["isodate"], + "module": self.__class__.__name__, + "event": f"whatsapp_msg_{record['direction']}", + "data": f"\"{text}\"" + } + + def check_indicators(self): + if not self.indicators: + return + + for message in self.results: + if not "data" in message: + continue + + message_links = check_for_links(message["data"]) + if self.indicators.check_domains(message_links): + self.detected.append(message) + + def _parse_db(self, db_path): + """Parse an Android msgstore.db WhatsApp database file. + :param db_path: Path to the Android WhatsApp database file to process + """ + conn = sqlite3.connect(db_path) + cur = conn.cursor() + cur.execute(""" + SELECT * FROM messages; + """) + names = [description[0] for description in cur.description] + + messages = [] + for item in cur: + message = dict() + for index, value in enumerate(item): + message[names[index]] = value + + if not message["data"]: + continue + + message["direction"] = ("send" if message["key_from_me"] == 1 else "received") + message["isodate"] = convert_timestamp_to_iso(message["timestamp"]) + + # If we find links in the messages or if they are empty we add them to the list. + if check_for_links(message["data"]) or message["data"].strip() == "": + messages.append(message) + + cur.close() + conn.close() + + log.info("Extracted a total of %d WhatsApp messages containing links", len(messages)) + self.results = messages + + def run(self): + try: + self._adb_process_file(os.path.join("/", WHATSAPP_PATH), self._parse_db) + except Exception as e: + self.log.error(e) diff --git a/mvt/android/modules/backup/__init__.py b/mvt/android/modules/backup/__init__.py new file mode 100644 index 0000000..138b300 --- /dev/null +++ b/mvt/android/modules/backup/__init__.py @@ -0,0 +1,8 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021 MVT Project Developers. +# See the file 'LICENSE' for usage and copying permissions, or find a copy at +# https://github.com/mvt-project/mvt/blob/main/LICENSE + +from .sms import SMS + +BACKUP_MODULES = [SMS,] diff --git a/mvt/android/modules/backup/sms.py b/mvt/android/modules/backup/sms.py new file mode 100644 index 0000000..86cc54b --- /dev/null +++ b/mvt/android/modules/backup/sms.py @@ -0,0 +1,64 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021 MVT Project Developers. +# See the file 'LICENSE' for usage and copying permissions, or find a copy at +# https://github.com/mvt-project/mvt/blob/main/LICENSE + +import os +import json +import zlib + +from mvt.common.module import MVTModule +from mvt.common.utils import check_for_links +from mvt.common.utils import convert_timestamp_to_iso + +class SMS(MVTModule): + + def __init__(self, file_path=None, base_folder=None, output_folder=None, + fast_mode=False, log=None, results=[]): + super().__init__(file_path=file_path, base_folder=base_folder, + output_folder=output_folder, fast_mode=fast_mode, + log=log, results=results) + + def check_indicators(self): + if not self.indicators: + return + + for message in self.results: + if not "body" in message: + continue + + message_links = check_for_links(message["body"]) + if self.indicators.check_domains(message_links): + self.detected.append(message) + + def _process_sms_file(self, file_path): + self.log.info("Processing SMS backup file at %s", file_path) + + with open(file_path, "rb") as handle: + data = zlib.decompress(handle.read()) + json_data = json.loads(data) + + for entry in json_data: + message_links = check_for_links(entry["body"]) + + # If we find links in the messages or if they are empty we add them to the list. + if message_links or entry["body"].strip() == "": + self.results.append(entry) + + def run(self): + app_folder = os.path.join(self.base_folder, + "apps", + "com.android.providers.telephony", + "d_f") + if not os.path.exists(app_folder): + raise FileNotFoundError("Unable to find the SMS backup folder") + + for file_name in os.listdir(app_folder): + if not file_name.endswith("_sms_backup"): + continue + + file_path = os.path.join(app_folder, file_name) + self._process_sms_file(file_path) + + self.log.info("Extracted a total of %d SMS messages containing links", + len(self.results)) diff --git a/mvt/common/__init__.py b/mvt/common/__init__.py new file mode 100644 index 0000000..2512a63 --- /dev/null +++ b/mvt/common/__init__.py @@ -0,0 +1,4 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021 MVT Project Developers. +# See the file 'LICENSE' for usage and copying permissions, or find a copy at +# https://github.com/mvt-project/mvt/blob/main/LICENSE diff --git a/mvt/common/indicators.py b/mvt/common/indicators.py new file mode 100644 index 0000000..9532295 --- /dev/null +++ b/mvt/common/indicators.py @@ -0,0 +1,175 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021 MVT Project Developers. +# See the file 'LICENSE' for usage and copying permissions, or find a copy at +# https://github.com/mvt-project/mvt/blob/main/LICENSE + +import os +import json + +from .url import URL + +class Indicators: + """This class is used to parse indicators from a STIX2 file and provide + functions to compare extracted artifacts to the indicators. + """ + + def __init__(self, file_path, log=None): + self.file_path = file_path + with open(self.file_path, "r") as handle: + self.data = json.load(handle) + + self.log = log + self.ioc_domains = [] + self.ioc_processes = [] + self.ioc_emails = [] + self.ioc_files = [] + self._parse_stix_file() + + def _parse_stix_file(self): + """Extract IOCs of given type from STIX2 definitions. + """ + for entry in self.data["objects"]: + try: + if entry["type"] != "indicator": + continue + except KeyError: + continue + + key, value = entry["pattern"].strip("[]").split("=") + value = value.strip("'") + + if key == "domain-name:value": + # We force domain names to lower case. + value = value.lower() + if value not in self.ioc_domains: + self.ioc_domains.append(value) + elif key == "process:name": + if value not in self.ioc_processes: + self.ioc_processes.append(value) + elif key == "email-addr:value": + # We force email addresses to lower case. + value = value.lower() + if value not in self.ioc_emails: + self.ioc_emails.append(value) + elif key == "file:name": + if value not in self.ioc_files: + self.ioc_files.append(value) + + def check_domain(self, url): + # TODO: If the IOC domain contains a subdomain, it is not currently + # being matched. + + try: + # First we use the provided URL. + orig_url = URL(url) + + if orig_url.check_if_shortened(): + # If it is, we try to retrieve the actual URL making an + # HTTP HEAD request. + unshortened = orig_url.unshorten() + + # self.log.info("Found a shortened URL %s -> %s", + # url, unshortened) + + # Now we check for any nested URL shorteners. + dest_url = URL(unshortened) + if dest_url.check_if_shortened(): + # self.log.info("Original URL %s appears to shorten another shortened URL %s ... checking!", + # orig_url.url, dest_url.url) + return self.check_domain(dest_url.url) + + final_url = dest_url + else: + # If it's not shortened, we just use the original URL object. + final_url = orig_url + except Exception as e: + # If URL parsing failed, we just try to do a simple substring + # match. + for ioc in self.ioc_domains: + if ioc.lower() in url: + self.log.warning("Maybe found a known suspicious domain: %s", url) + return True + + # If nothing matched, we can quit here. + return False + + # If all parsing worked, we start walking through available domain indicators. + for ioc in self.ioc_domains: + # First we check the full domain. + if final_url.domain.lower() == ioc: + if orig_url.is_shortened and orig_url.url != final_url.url: + self.log.warning("Found a known suspicious domain %s shortened as %s", + final_url.url, orig_url.url) + else: + self.log.warning("Found a known suspicious domain: %s", final_url.url) + + return True + + # Then we just check the top level domain. + if final_url.top_level.lower() == ioc: + if orig_url.is_shortened and orig_url.url != final_url.url: + self.log.warning("Found a sub-domain matching a suspicious top level %s shortened as %s", + final_url.url, orig_url.url) + else: + self.log.warning("Found a sub-domain matching a suspicious top level: %s", final_url.url) + + return True + + def check_domains(self, urls): + """Check the provided list of (suspicious) domains against a list of URLs. + :param urls: List of URLs to check + """ + for url in urls: + if self.check_domain(url): + return True + + def check_process(self, process): + """Check the provided process name against the list of process + indicators. + :param process: Process name to check + """ + if not process: + return False + + proc_name = os.path.basename(process) + if proc_name in self.ioc_processes: + self.log.warning("Found a known suspicious process name \"%s\"", process) + return True + + if len(proc_name) == 16: + for bad_proc in self.ioc_processes: + if bad_proc.startswith(proc_name): + self.log.warning("Found a truncated known suspicious process name \"%s\"", process) + return True + + def check_processes(self, processes): + """Check the provided list of processes against the list of + process indicators. + :param processes: List of processes to check + """ + for process in processes: + if self.check_process(process): + return True + + def check_email(self, email): + """Check the provided email against the list of email indicators. + :param email: Suspicious email to check + """ + if not email: + return False + + if email.lower() in self.ioc_emails: + self.log.warning("Found a known suspicious email address: \"%s\"", email) + return True + + def check_file(self, file_path): + """Check the provided file path against the list of file indicators. + :param file_path: Path or name of the file to check + """ + if not file_path: + return False + + file_name = os.path.basename(file_path) + if file_name in self.ioc_files: + self.log.warning("Found a known suspicious file: \"%s\"", file_path) + return True diff --git a/mvt/common/module.py b/mvt/common/module.py new file mode 100644 index 0000000..378acf5 --- /dev/null +++ b/mvt/common/module.py @@ -0,0 +1,172 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021 MVT Project Developers. +# See the file 'LICENSE' for usage and copying permissions, or find a copy at +# https://github.com/mvt-project/mvt/blob/main/LICENSE + +import os +import re +import csv +import glob +import logging +import simplejson as json + +from .indicators import Indicators + +class MVTModule(object): + """This class provides a base for all extraction modules.""" + + enabled = True + slug = None + + def __init__(self, file_path=None, base_folder=None, output_folder=None, + fast_mode=False, log=None, results=[]): + """Initialize module. + :param file_path: Path to the module's database file, if there is any. + :param base_folder: Path to the base folder (backup or filesystem dump) + :param output_folder: Folder where results will be stored + :param fast_mode: Flag to enable or disable slow modules + :param log: Handle to logger + :param results: Provided list of results entries + """ + self.file_path = file_path + self.base_folder = base_folder + self.output_folder = output_folder + self.fast_mode = fast_mode + self.log = log + self.indicators = None + self.results = results + self.detected = [] + self.timeline = [] + self.timeline_detected = [] + + @classmethod + def from_json(cls, json_path, log=None): + with open(json_path, "r") as handle: + results = json.load(handle) + if log: + log.info("Loaded %d results from \"%s\"", + len(results), json_path) + return cls(results=results, log=log) + + def get_slug(self): + if self.slug: + return self.slug + + sub = re.sub("(.)([A-Z][a-z]+)", r"\1_\2", self.__class__.__name__) + return re.sub("([a-z0-9])([A-Z])", r"\1_\2", sub).lower() + + def _find_paths(self, root_paths): + for root_path in root_paths: + for found_path in glob.glob(os.path.join(self.base_folder, root_path)): + if not os.path.exists(found_path): + continue + yield found_path + + def load_indicators(self, file_path): + self.indicators = Indicators(file_path, self.log) + + def check_indicators(self): + """Check the results of this module againt a provided list of + indicators.""" + raise NotImplementedError + + def save_to_json(self): + """Save the collected results to a json file. + """ + if not self.output_folder: + return + + name = self.get_slug() + + if self.results: + results_file_name = f"{name}.json" + results_json_path = os.path.join(self.output_folder, results_file_name) + with open(results_json_path, "w") as handle: + json.dump(self.results, handle, indent=4) + + if self.detected: + detected_file_name = f"{name}_detected.json" + detected_json_path = os.path.join(self.output_folder, detected_file_name) + with open(detected_json_path, "w") as handle: + json.dump(self.detected, handle, indent=4) + + def serialize(self, record): + raise NotImplementedError + + def to_timeline(self): + """Convert results into a timeline. + """ + for result in self.results: + record = self.serialize(result) + if type(record) == list: + self.timeline.extend(record) + else: + self.timeline.append(record) + + for detected in self.detected: + record = self.serialize(detected) + if type(record) == list: + self.timeline_detected.extend(record) + else: + self.timeline_detected.append(record) + + # De-duplicate timeline entries + self.timeline = self.timeline_deduplicate(self.timeline) + self.timeline_detected = self.timeline_deduplicate(self.timeline_detected) + + def timeline_deduplicate(self, timeline): + """Serialize entry as JSON to deduplicate repeated entries""" + timeline_set = set() + for record in timeline: + timeline_set.add(json.dumps(record, sort_keys=True)) + return [json.loads(record) for record in timeline_set] + + def run(self): + """Run the main module procedure. + """ + raise NotImplementedError + + +def run_module(module): + module.log.info("Running module %s...", module.__class__.__name__) + + try: + module.run() + except NotImplementedError: + module.log.exception("The run() procedure of module %s was not implemented yet!", + module.__class__.__name__) + except FileNotFoundError as e: + module.log.error("There might be no data to extract by module %s: %s", + module.__class__.__name__, e) + except Exception as e: + module.log.exception("Error in running extraction from module %s: %s", + module.__class__.__name__, e) + else: + try: + module.check_indicators() + except NotImplementedError: + pass + + try: + module.to_timeline() + except NotImplementedError: + pass + + module.save_to_json() + + +def save_timeline(timeline, timeline_path): + """Save the timeline in a csv file. + :param timeline: List of records to order and store. + :param timeline_path: Path to the csv file to store the timeline to. + """ + with open(timeline_path, "a+") as handle: + csvoutput = csv.writer(handle, delimiter=",", quotechar="\"") + csvoutput.writerow(["UTC Timestamp", "Plugin", "Event", "Description"]) + for event in sorted(timeline, key=lambda x: x["timestamp"] if x["timestamp"] is not None else ""): + csvoutput.writerow([ + event["timestamp"], + event["module"], + event["event"], + event["data"], + ]) diff --git a/mvt/common/options.py b/mvt/common/options.py new file mode 100644 index 0000000..bc0842b --- /dev/null +++ b/mvt/common/options.py @@ -0,0 +1,39 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021 MVT Project Developers. +# See the file 'LICENSE' for usage and copying permissions, or find a copy at +# https://github.com/mvt-project/mvt/blob/main/LICENSE + +# From: https://gist.github.com/stanchan/bce1c2d030c76fe9223b5ff6ad0f03db + +from click import command, option, Option, UsageError + +class MutuallyExclusiveOption(Option): + """This class extends click to support mutually exclusive options. + """ + + def __init__(self, *args, **kwargs): + self.mutually_exclusive = set(kwargs.pop("mutually_exclusive", [])) + help = kwargs.get("help", "") + if self.mutually_exclusive: + ex_str = ", ".join(self.mutually_exclusive) + kwargs["help"] = help + ( + " NOTE: This argument is mutually exclusive with " + "arguments: [" + ex_str + "]." + ) + super(MutuallyExclusiveOption, self).__init__(*args, **kwargs) + + def handle_parse_result(self, ctx, opts, args): + if self.mutually_exclusive.intersection(opts) and self.name in opts: + raise UsageError( + "Illegal usage: `{}` is mutually exclusive with " + "arguments `{}`.".format( + self.name, + ", ".join(self.mutually_exclusive) + ) + ) + + return super(MutuallyExclusiveOption, self).handle_parse_result( + ctx, + opts, + args + ) diff --git a/mvt/common/url.py b/mvt/common/url.py new file mode 100644 index 0000000..57c7723 --- /dev/null +++ b/mvt/common/url.py @@ -0,0 +1,295 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021 MVT Project Developers. +# See the file 'LICENSE' for usage and copying permissions, or find a copy at +# https://github.com/mvt-project/mvt/blob/main/LICENSE + +import requests +from tld import get_tld + +SHORTENER_DOMAINS = [ + "1link.in", + "1url.com", + "2big.at", + "2pl.us", + "2tu.us", + "2ya.com", + "4url.cc", + "6url.com", + "a2a.me", + "abbrr.com", + "adf.ly", + "adjix.com", + "a.gg", + "alturl.com", + "a.nf", + "atu.ca", + "b23.ru", + "bacn.me", + "bit.ly", + "bit.do", + "bkite.com", + "bloat.me", + "budurl.com", + "buff.ly", + "buk.me", + "burnurl.com", + "chilp.it", + "clck.ru", + "clickmeter.com", + "cli.gs", + "c-o.in", + "cort.as", + "cut.ly", + "cuturl.com", + "decenturl.com", + "decenturl.com", + "dfl8.me", + "digbig.com", + "digg.com", + "doiop.com", + "dwarfurl.com", + "dy.fi", + "easyuri.com", + "easyurl.net", + "eepurl.com", + "esyurl.com", + "ewerl.com", + "fa.b", + "fff.to", + "ff.im", + "fhurl.com", + "fire.to", + "firsturl.de", + "flic.kr", + "fly2.ws", + "fon.gs", + "fwd4.me", + "gl.am", + "go2cut.com", + "go2.me", + "go.9nl.com", + "goo.gl", + "goshrink.com", + "gowat.ch", + "gri.ms", + "gurl.es", + "hellotxt.com", + "hex.io", + "hover.com", + "href.in", + "htxt.it", + "hugeurl.com", + "hurl.it", + "hurl.me", + "hurl.ws", + "icanhaz.com", + "idek.net", + "inreply.to", + "iscool.net", + "is.gd", + "iterasi.net", + "jijr.com", + "jmp2.net", + "just.as", + "kissa.be", + "kl.am", + "klck.me", + "korta.nu", + "krunchd.com", + "liip.to", + "liltext.com", + "lin.cr", + "linkbee.com", + "linkbun.ch", + "liurl.cn", + "lnk.gd", + "lnk.in", + "ln-s.net", + "ln-s.ru", + "loopt.us", + "lru.jp", + "lt.tl", + "lurl.no", + "metamark.net", + "migre.me", + "minilien.com", + "miniurl.com", + "minurl.fr", + "moourl.com", + "myurl.in", + "ne1.net", + "njx.me", + "nn.nf", + "notlong.com", + "nsfw.in", + "om.ly", + "ow.ly", + "o-x.fr", + "pd.am", + "pic.gd", + "ping.fm", + "piurl.com", + "pnt.me", + "poprl.com", + "posted.at", + "post.ly", + "profile.to", + "qicute.com", + "qlnk.net", + "quip-art.com", + "rb6.me", + "redirx.com", + "rickroll.it", + "ri.ms", + "riz.gd", + "rsmonkey.com", + "rubyurl.com", + "ru.ly", + "s7y.us", + "safe.mn", + "sharein.com", + "sharetabs.com", + "shorl.com", + "short.ie", + "shortlinks.co.uk", + "shortna.me", + "short.to", + "shorturl.com", + "shoturl.us", + "shrinkify.com", + "shrinkster.com", + "shrten.com", + "shrt.st", + "shrunkin.com", + "shw.me", + "simurl.com", + "sn.im", + "snipr.com", + "snipurl.com", + "snurl.com", + "sp2.ro", + "spedr.com", + "sqrl.it", + "starturl.com", + "sturly.com", + "su.pr", + "t.co", + "tcrn.ch", + "thrdl.es", + "tighturl.com", + "tiny123.com", + "tinyarro.ws", + "tiny.cc", + "tiny.pl", + "tinytw.it", + "tinyuri.ca", + "tinyurl.com", + "tinyvid.io", + "tnij.org", + "togoto.us", + "to.ly", + "traceurl.com", + "tr.im", + "tr.my", + "turo.us", + "tweetburner.com", + "twirl.at", + "twit.ac", + "twitterpan.com", + "twitthis.com", + "twiturl.de", + "twurl.cc", + "twurl.nl", + "u6e.de", + "ub0.cc", + "u.mavrev.com", + "u.nu", + "updating.me", + "ur1.ca", + "url4.eu", + "urlao.com", + "urlbrief.com", + "url.co.uk", + "urlcover.com", + "urlcut.com", + "urlenco.de", + "urlhawk.com", + "url.ie", + "urlkiss.com", + "urlot.com", + "urlpire.com", + "urlx.ie", + "urlx.org", + "urlzen.com", + "virl.com", + "vl.am", + "w3t.org", + "wapurl.co.uk", + "wipi.es", + "wp.me", + "xaddr.com", + "x.co", + "xeeurl.com", + "xr.com", + "xrl.in", + "xrl.us", + "x.se", + "xurl.jp", + "xzb.cc", + "yep.it", + "yfrog.com", + "yweb.com", + "zi.ma", + "zi.pe", + "zipmyurl.com", + "zz.gd", + "ymlp.com", + "forms.gle", + "ht.ly", + "lnkd.in", + "1drv.ms", +] + +class URL: + + def __init__(self, url): + if type(url) == bytes: + url = url.decode() + + self.url = url + self.domain = self.get_domain() + self.top_level = self.get_top_level() + self.is_shortened = False + + def get_domain(self): + """Get the domain from a URL. + :param url: URL to parse + :returns: Just the domain name extracted from the URL + """ + # TODO: Properly handle exception. + try: + return get_tld(self.url, as_object=True, fix_protocol=True).parsed_url.netloc.lower().lstrip("www.") + except: + return None + + def get_top_level(self): + """Get only the top level domain from a URL. + :param url: URL to parse + :returns: The top level domain extracted from the URL + """ + # TODO: Properly handle exception. + try: + return get_tld(self.url, as_object=True, fix_protocol=True).fld.lower() + except: + return None + + def check_if_shortened(self): + if self.domain.lower() in SHORTENER_DOMAINS: + self.is_shortened = True + + return self.is_shortened + + def unshorten(self): + res = requests.head(self.url) + if str(res.status_code).startswith("30"): + return res.headers["Location"] diff --git a/mvt/common/utils.py b/mvt/common/utils.py new file mode 100644 index 0000000..6805704 --- /dev/null +++ b/mvt/common/utils.py @@ -0,0 +1,98 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021 MVT Project Developers. +# See the file 'LICENSE' for usage and copying permissions, or find a copy at +# https://github.com/mvt-project/mvt/blob/main/LICENSE + +import os +import re +import datetime +import hashlib + +def convert_mactime_to_unix(timestamp, from_2001=True): + """Converts Mac Standard Time to a Unix timestamp. + :param timestamp: MacTime timestamp (either int or float) + :returns: Unix epoch timestamp + """ + if not timestamp: + return None + + # This is to fix formats in case of, for example, SMS messages database + # timestamp format. + if type(timestamp) == int and len(str(timestamp)) == 18: + timestamp = int(str(timestamp)[:9]) + + # MacTime counts from 2001-01-01. + if from_2001: + timestamp = timestamp + 978307200 + + # TODO: This is rather ugly. Happens sometimes with invalid timestamps. + try: + return datetime.datetime.utcfromtimestamp(timestamp) + except Exception: + return None + + +def convert_chrometime_to_unix(timestamp): + """Converts Chrome timestamp to a Unix timestamp. + :param timestamp: Chrome timestamp as int + :returns: Unix epoch timestamp + """ + epoch_start = datetime.datetime(1601, 1 , 1) + delta = datetime.timedelta(microseconds=timestamp) + return epoch_start + delta + + +def convert_timestamp_to_iso(timestamp): + """Converts Unix timestamp to ISO string. + :param timestamp: Unix timestamp + :returns: ISO timestamp string in YYYY-mm-dd HH:MM:SS.ms format + """ + try: + return timestamp.strftime("%Y-%m-%d %H:%M:%S.%f") + except Exception: + return None + +def check_for_links(text): + """Checks if a given text contains HTTP links. + :param text: Any provided text + :returns: Search results + """ + return re.findall("(?Phttps?://[^\s]+)", text, re.IGNORECASE) + +def get_sha256_from_file_path(file_path): + """Calculate the SHA256 hash of a file from a file path. + :param file_path: Path to the file to hash + :returns: The SHA256 hash string + """ + sha256_hash = hashlib.sha256() + with open(file_path, "rb") as handle: + for byte_block in iter(lambda: handle.read(4096), b""): + sha256_hash.update(byte_block) + + return sha256_hash.hexdigest() + +# Note: taken from here: +# https://stackoverflow.com/questions/57014259/json-dumps-on-dictionary-with-bytes-for-keys +def keys_bytes_to_string(obj): + """Convert object keys from bytes to string. + :param obj: Object to convert from bytes to string. + :returns: Converted object. + """ + new_obj = {} + if not isinstance(obj, dict): + if isinstance(obj, (tuple, list, set)): + value = [keys_bytes_to_string(x) for x in obj] + return value + else: + return obj + + for key, value in obj.items(): + if isinstance(key, bytes): + key = key.decode() + if isinstance(value, dict): + value = keys_bytes_to_string(value) + elif isinstance(value, (tuple, list, set)): + value = [keys_bytes_to_string(x) for x in value] + new_obj[key] = value + + return new_obj diff --git a/mvt/ios/__init__.py b/mvt/ios/__init__.py new file mode 100644 index 0000000..2ae7e01 --- /dev/null +++ b/mvt/ios/__init__.py @@ -0,0 +1,6 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021 MVT Project Developers. +# See the file 'LICENSE' for usage and copying permissions, or find a copy at +# https://github.com/mvt-project/mvt/blob/main/LICENSE + +from .cli import cli diff --git a/mvt/ios/cli.py b/mvt/ios/cli.py new file mode 100644 index 0000000..67b2864 --- /dev/null +++ b/mvt/ios/cli.py @@ -0,0 +1,215 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021 MVT Project Developers. +# See the file 'LICENSE' for usage and copying permissions, or find a copy at +# https://github.com/mvt-project/mvt/blob/main/LICENSE + +import os +import sys +import click +import tarfile +import logging +from rich.logging import RichHandler + +from mvt.common.module import run_module, save_timeline +from mvt.common.options import MutuallyExclusiveOption +from mvt.common.indicators import Indicators + +from .decrypt import DecryptBackup +from .modules.fs import BACKUP_MODULES, FS_MODULES + +# Setup logging using Rich. +LOG_FORMAT = "[%(name)s] %(message)s" +logging.basicConfig(level="INFO", format=LOG_FORMAT, handlers=[ + RichHandler(show_path=False, log_time_format="%X")]) +log = logging.getLogger(__name__) + +# Help messages of repeating options. +OUTPUT_HELP_MESSAGE = "Specify a path to a folder where you want to store JSON results" + + +#============================================================================== +# Main +#============================================================================== +@click.group(invoke_without_command=False) +def cli(): + return + + +#============================================================================== +# Command: decrypt-backup +#============================================================================== +@cli.command("decrypt-backup", help="Decrypt an encrypted iTunes backup") +@click.option("--destination", "-d", required=True, + help="Path to the folder where to store the decrypted backup") +@click.option("--password", "-p", cls=MutuallyExclusiveOption, + help="Password to use to decrypt the backup", + mutually_exclusive=["key_file"]) +@click.option("--key-file", "-k", cls=MutuallyExclusiveOption, + type=click.Path(exists=True), + help="File containing raw encryption key to use to decrypt the backup", + mutually_exclusive=["password"]) +@click.argument("BACKUP_PATH", type=click.Path(exists=True)) +def decrypt_backup(destination, password, key_file, backup_path): + backup = DecryptBackup(backup_path, destination) + if password: + backup.decrypt_with_password(password) + elif key_file: + backup.decrypt_with_key_file(key_file) + else: + raise click.ClickException("Missing required option. Specify either " + "--password or --key-file.") + + +#============================================================================== +# Command: check-backup +#============================================================================== +@cli.command("check-backup", help="Extract artifacts from an iTunes backup") +@click.option("--iocs", "-i", type=click.Path(exists=True), help="Path to indicators file") +@click.option("--output", "-o", type=click.Path(exists=True), help=OUTPUT_HELP_MESSAGE) +@click.option("--fast", "-f", is_flag=True, help="Avoid running time/resource consuming features") +@click.option("--list-modules", "-l", is_flag=True, help="Print list of available modules and exit") +@click.option("--module", "-m", help="Name of a single module you would like to run instead of all") +@click.argument("BACKUP_PATH", type=click.Path(exists=True)) +def check_backup(iocs, output, fast, backup_path, list_modules, module): + if list_modules: + log.info("Following is the list of available check-backup modules:") + for backup_module in BACKUP_MODULES: + log.info(" - %s", backup_module.__name__) + + return + + log.info("Checking iTunes backup located at: %s", backup_path) + + if iocs: + # Pre-load indicators for performance reasons. + log.info("Loading indicators from provided file at: %s", iocs) + indicators = Indicators(iocs) + + timeline = [] + timeline_detected = [] + for backup_module in BACKUP_MODULES: + if module and backup_module.__name__ != module: + continue + + m = backup_module(base_folder=backup_path, output_folder=output, fast_mode=fast, + log=logging.getLogger(backup_module.__module__)) + m.is_backup = True + + if iocs: + indicators.log = m.log + m.indicators = indicators + + run_module(m) + timeline.extend(m.timeline) + timeline_detected.extend(m.timeline_detected) + + if output: + if len(timeline) > 0: + save_timeline(timeline, os.path.join(output, "timeline.csv")) + if len(timeline_detected) > 0: + save_timeline(timeline_detected, os.path.join(output, "timeline_detected.csv")) + + +#============================================================================== +# Command: check-fs +#============================================================================== +@cli.command("check-fs", help="Extract artifacts from a full filesystem dump") +@click.option("--iocs", "-i", type=click.Path(exists=True), help="Path to indicators file") +@click.option("--output", "-o", type=click.Path(exists=True), help=OUTPUT_HELP_MESSAGE) +@click.option("--fast", "-f", is_flag=True, help="Avoid running time/resource consuming features") +@click.option("--list-modules", "-l", is_flag=True, help="Print list of available modules and exit") +@click.option("--module", "-m", help="Name of a single module you would like to run instead of all") +@click.argument("DUMP_PATH", type=click.Path(exists=True)) +def check_fs(iocs, output, fast, dump_path, list_modules, module): + if list_modules: + log.info("Following is the list of available check-fs modules:") + for fs_module in FS_MODULES: + log.info(" - %s", fs_module.__name__) + + return + + log.info("Checking filesystem dump located at: %s", dump_path) + + if iocs: + # Pre-load indicators for performance reasons. + log.info("Loading indicators from provided file at: %s", iocs) + indicators = Indicators(iocs) + + timeline = [] + timeline_detected = [] + for fs_module in FS_MODULES: + if module and fs_module.__name__ != module: + continue + + m = fs_module(base_folder=dump_path, output_folder=output, fast_mode=fast, + log=logging.getLogger(fs_module.__module__)) + + m.is_fs_dump = True + + if iocs: + indicators.log = m.log + m.indicators = indicators + + run_module(m) + timeline.extend(m.timeline) + timeline_detected.extend(m.timeline_detected) + + if output: + if len(timeline) > 0: + save_timeline(timeline, os.path.join(output, "timeline.csv")) + if len(timeline_detected) > 0: + save_timeline(timeline_detected, os.path.join(output, "timeline_detected.csv")) + + +#============================================================================== +# Command: check-iocs +#============================================================================== +@cli.command("check-iocs", help="Compare stored JSON results to provided indicators") +@click.option("--iocs", "-i", required=True, type=click.Path(exists=True), + help="Path to indicators file") +@click.option("--list-modules", "-l", is_flag=True, help="Print list of available modules and exit") +@click.option("--module", "-m", help="Name of a single module you would like to run instead of all") +@click.argument("FOLDER", type=click.Path(exists=True)) +def check_iocs(iocs, list_modules, module, folder): + all_modules = [] + for entry in BACKUP_MODULES + FS_MODULES + SYSDIAGNOSE_MODULES: + if entry not in all_modules: + all_modules.append(entry) + + if list_modules: + log.info("Following is the list of available check-iocs modules:") + for iocs_module in all_modules: + log.info(" - %s", iocs_module.__name__) + + return + + log.info("Checking stored results against provided indicators...") + + # Pre-load indicators for performance reasons. + log.info("Loading indicators from provided file at: %s", iocs) + indicators = Indicators(iocs) + + for file_name in os.listdir(folder): + name_only, ext = os.path.splitext(file_name) + file_path = os.path.join(folder, file_name) + + for iocs_module in all_modules: + if module and iocs_module.__name__ != module: + continue + + if iocs_module().get_slug() != name_only: + continue + + log.info("Loading results from \"%s\" with module %s", file_name, + iocs_module.__name__) + + m = iocs_module.from_json(file_path, + log=logging.getLogger(iocs_module.__module__)) + + indicators.log = m.log + m.indicators = indicators + + try: + m.check_indicators() + except NotImplementedError: + continue diff --git a/mvt/ios/decrypt.py b/mvt/ios/decrypt.py new file mode 100644 index 0000000..7b1acb0 --- /dev/null +++ b/mvt/ios/decrypt.py @@ -0,0 +1,113 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021 MVT Project Developers. +# See the file 'LICENSE' for usage and copying permissions, or find a copy at +# https://github.com/mvt-project/mvt/blob/main/LICENSE + +import os +import shutil +import sqlite3 +import logging +import binascii +from iOSbackup import iOSbackup + +log = logging.getLogger(__name__) + +class DecryptBackup: + """This class provides functions to decrypt an encrypted iTunes backup + using either a password or a key file. + """ + + def __init__(self, backup_path, dest_path): + """Decrypts an encrypted iOS backup. + :param backup_path: Path to the encrypted backup folder + :param dest_path: Path to the folder where to store the decrypted backup + """ + self.backup_path = backup_path + self.dest_path = dest_path + self._backup = None + + def _process_backup(self): + manifest_path = os.path.join(self.dest_path, "Manifest.db") + # We extract a decrypted Manifest.db. + self._backup.getManifestDB() + # We store it to the destination folder. + shutil.copy(self._backup.manifestDB, manifest_path) + + for item in self._backup.getBackupFilesList(): + try: + file_id = item["backupFile"] + relative_path = item["relativePath"] + domain = item["domain"] + + # This may be a partial backup. Skip files from the manifest which do not exist locally. + source_file_path = os.path.join(self.backup_path, file_id[0:2], file_id) + if not os.path.exists(source_file_path): + log.debug("Skipping file %s. File not found in encrypted backup directory.", + source_file_path) + continue + + item_folder = os.path.join(self.dest_path, file_id[0:2]) + if not os.path.exists(item_folder): + os.makedirs(item_folder) + + # iOSBackup getFileDecryptedCopy() claims to read a "file" parameter + # but the code actually is reading the "manifest" key. + # Add manifest plist to both keys to handle this. + item["manifest"] = item["file"] + + self._backup.getFileDecryptedCopy(manifestEntry=item, + targetName=file_id, + targetFolder=item_folder) + log.info("Decrypted file %s [%s] to %s/%s", relative_path, domain, item_folder, file_id) + except Exception as e: + log.error("Failed to decrypt file %s: %s", relative_path, e) + + def decrypt_with_password(self, password): + """Decrypts an encrypted iOS backup. + :param password: Password to use to decrypt the original backup + """ + log.info("Decrypting iOS backup at path %s with password", self.backup_path) + + if not os.path.exists(self.dest_path): + os.makedirs(self.dest_path) + + try: + self._backup = iOSbackup(udid=os.path.basename(self.backup_path), + cleartextpassword=password, + backuproot=os.path.dirname(self.backup_path)) + except Exception as e: + log.exception(e) + log.critical("Failed to decrypt backup. Did you provide the correct password?") + return + else: + self._process_backup() + + def decrypt_with_key_file(self, key_file): + """Decrypts an encrypted iOS backup using a key file. + :param key_file: File to read the key bytes to decrypt the backup + """ + log.info("Decrypting iOS backup at path %s with key file %s", + self.backup_path, key_file) + + if not os.path.exists(self.dest_path): + os.makedirs(self.dest_path) + + with open(key_file, "rb") as handle: + key_bytes = handle.read() + + # Key should be 64 hex encoded characters (32 raw bytes) + if len(key_bytes) != 64: + log.critical("Invalid key from key file. Did you provide the correct key file?") + return + + try: + key_bytes_raw = binascii.unhexlify(key_bytes) + self._backup = iOSbackup(udid=os.path.basename(self.backup_path), + derivedkey=key_bytes_raw, + backuproot=os.path.dirname(self.backup_path)) + except Exception as e: + log.exception(e) + log.critical("Failed to decrypt backup. Did you provide the correct key file?") + return + else: + self._process_backup() diff --git a/mvt/ios/modules/__init__.py b/mvt/ios/modules/__init__.py new file mode 100644 index 0000000..2512a63 --- /dev/null +++ b/mvt/ios/modules/__init__.py @@ -0,0 +1,4 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021 MVT Project Developers. +# See the file 'LICENSE' for usage and copying permissions, or find a copy at +# https://github.com/mvt-project/mvt/blob/main/LICENSE diff --git a/mvt/ios/modules/fs/__init__.py b/mvt/ios/modules/fs/__init__.py new file mode 100644 index 0000000..dad5124 --- /dev/null +++ b/mvt/ios/modules/fs/__init__.py @@ -0,0 +1,42 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021 MVT Project Developers. +# See the file 'LICENSE' for usage and copying permissions, or find a copy at +# https://github.com/mvt-project/mvt/blob/main/LICENSE + +from .manifest import Manifest +from .contacts import Contacts +from .net_netusage import Netusage +from .net_datausage import Datausage +from .safari_history import SafariHistory +from .safari_favicon import SafariFavicon +from .safari_browserstate import SafariBrowserState +from .webkit_indexeddb import WebkitIndexedDB +from .webkit_localstorage import WebkitLocalStorage +from .webkit_safariviewservice import WebkitSafariViewService +from .webkit_session_resource_log import WebkitSessionResourceLog +from .chrome_history import ChromeHistory +from .chrome_favicon import ChromeFavicon +from .firefox_history import FirefoxHistory +from .firefox_favicon import FirefoxFavicon +from .version_history import IOSVersionHistory +from .idstatuscache import IDStatusCache +from .locationd import LocationdClients +from .interactionc import InteractionC +from .sms import SMS +from .sms_attachments import SMSAttachments +from .calls import Calls +from .whatsapp import Whatsapp +from .cache_files import CacheFiles +from .filesystem import Filesystem + +BACKUP_MODULES = [SafariBrowserState, SafariHistory, Datausage, SMS, SMSAttachments, + ChromeHistory, ChromeFavicon, WebkitSessionResourceLog, + Calls, IDStatusCache, LocationdClients, InteractionC, + FirefoxHistory, FirefoxFavicon, Contacts, Manifest, Whatsapp] + +FS_MODULES = [IOSVersionHistory, SafariHistory, SafariFavicon, SafariBrowserState, + WebkitIndexedDB, WebkitLocalStorage, WebkitSafariViewService, + WebkitSessionResourceLog, Datausage, Netusage, ChromeHistory, + ChromeFavicon, Calls, IDStatusCache, SMS, SMSAttachments, + LocationdClients, InteractionC, FirefoxHistory, FirefoxFavicon, + Contacts, CacheFiles, Whatsapp, Filesystem] diff --git a/mvt/ios/modules/fs/base.py b/mvt/ios/modules/fs/base.py new file mode 100644 index 0000000..7a5437d --- /dev/null +++ b/mvt/ios/modules/fs/base.py @@ -0,0 +1,55 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021 MVT Project Developers. +# See the file 'LICENSE' for usage and copying permissions, or find a copy at +# https://github.com/mvt-project/mvt/blob/main/LICENSE + +import os +import glob + +from mvt.common.module import MVTModule + +class IOSExtraction(MVTModule): + """This class provides a base for all iOS filesystem/backup extraction modules.""" + + is_backup = False + is_fs_dump = False + is_sysdiagnose = False + + def _find_ios_database(self, backup_ids=None, root_paths=[]): + """Try to locate the module's database file from either an iTunes + backup or a full filesystem dump. + :param backup_id: iTunes backup database file's ID (or hash). + """ + file_path = None + # First we check if the was an explicit file path specified. + if not self.file_path: + # If not, we first try with backups. + # We construct the path to the file according to the iTunes backup + # folder structure, if we have a valid ID. + if backup_ids: + for backup_id in backup_ids: + file_path = os.path.join(self.base_folder, backup_id[0:2], backup_id) + # If we found the correct backup file, then we stop searching. + if os.path.exists(file_path): + break + + # If this file does not exist we might be processing a full + # filesystem dump (checkra1n all the things!). + if not file_path or not os.path.exists(file_path): + # We reset the file_path. + file_path = None + for root_path in root_paths: + for found_path in glob.glob(os.path.join(self.base_folder, root_path)): + # If we find a valid path, we set file_path. + if os.path.exists(found_path): + file_path = found_path + break + + # Otherwise, we reset the file_path again. + file_path = None + + # If we do not find any, we fail. + if file_path: + self.file_path = file_path + else: + raise FileNotFoundError("Unable to find the module's database file") diff --git a/mvt/ios/modules/fs/cache_files.py b/mvt/ios/modules/fs/cache_files.py new file mode 100644 index 0000000..ef35931 --- /dev/null +++ b/mvt/ios/modules/fs/cache_files.py @@ -0,0 +1,77 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021 MVT Project Developers. +# See the file 'LICENSE' for usage and copying permissions, or find a copy at +# https://github.com/mvt-project/mvt/blob/main/LICENSE + +import os +import sqlite3 + +from .base import IOSExtraction + +class CacheFiles(IOSExtraction): + + def __init__(self, file_path=None, base_folder=None, output_folder=None, + fast_mode=False, log=None, results=[]): + super().__init__(file_path=file_path, base_folder=base_folder, + output_folder=output_folder, fast_mode=fast_mode, + log=log, results=results) + + def serialize(self, record): + records = [] + for item in self.results[record]: + records.append({ + "timestamp": item["isodate"], + "module": self.__class__.__name__, + "event": "cache_response", + "data": f"{record} recorded visit to URL {item['url']}" + }) + + return records + + def check_indicators(self): + if not self.indicators: + return + + self.detected = {} + for key, items in self.results.items(): + for item in items: + if self.indicators.check_domain(item["url"]): + if key not in self.detected: + self.detected[key] = [item,] + else: + self.detected[key].append(item) + + def _process_cache_file(self, file_path): + self.log.info("Processing cache file at path: %s", file_path) + + conn = sqlite3.connect(file_path) + cur = conn.cursor() + + try: + cur.execute("SELECT * FROM cfurl_cache_response;") + except sqlite3.OperationalError: + return + + key_name = os.path.relpath(file_path, self.base_folder) + if not key_name in self.results: + self.results[key_name] = [] + + for row in cur: + self.results[key_name].append(dict( + entry_id=row[0], + version=row[1], + hash_value=row[2], + storage_policy=row[3], + url=row[4], + isodate=row[5], + )) + + def run(self): + self.results = {} + for root, dirs, files in os.walk(self.base_folder): + for file_name in files: + if file_name != "Cache.db": + continue + + file_path = os.path.join(root, file_name) + self._process_cache_file(file_path) diff --git a/mvt/ios/modules/fs/calls.py b/mvt/ios/modules/fs/calls.py new file mode 100644 index 0000000..2ed3de5 --- /dev/null +++ b/mvt/ios/modules/fs/calls.py @@ -0,0 +1,60 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021 MVT Project Developers. +# See the file 'LICENSE' for usage and copying permissions, or find a copy at +# https://github.com/mvt-project/mvt/blob/main/LICENSE + +import sqlite3 + +from mvt.common.utils import convert_mactime_to_unix, convert_timestamp_to_iso +from .base import IOSExtraction + +CALLS_BACKUP_IDS = [ + "5a4935c78a5255723f707230a451d79c540d2741", +] +CALLS_ROOT_PATHS = [ + "private/var/mobile/Library/CallHistoryDB/CallHistory.storedata" +] + +class Calls(IOSExtraction): + """This module extracts phone calls details""" + + def __init__(self, file_path=None, base_folder=None, output_folder=None, + fast_mode=False, log=None, results=[]): + super().__init__(file_path=file_path, base_folder=base_folder, + output_folder=output_folder, fast_mode=fast_mode, + log=log, results=results) + + def serialize(self, record): + return { + "timestamp": record["isodate"], + "module": self.__class__.__name__, + "event": "call", + "data": f"From {record['number']} using {record['provider']} during {record['duration']} seconds" + } + + def run(self): + self._find_ios_database(backup_ids=CALLS_BACKUP_IDS, root_paths=CALLS_ROOT_PATHS) + self.log.info("Found Calls database at path: %s", self.file_path) + + conn = sqlite3.connect(self.file_path) + cur = conn.cursor() + cur.execute(""" + SELECT + ZDATE, ZDURATION, ZLOCATION, ZADDRESS, ZSERVICE_PROVIDER + FROM ZCALLRECORD; + """) + names = [description[0] for description in cur.description] + + for entry in cur: + self.results.append({ + "isodate": convert_timestamp_to_iso(convert_mactime_to_unix(entry[0])), + "duration": entry[1], + "location": entry[2], + "number": entry[3].decode("utf-8") if entry[3] and entry[3] is bytes else entry[3], + "provider": entry[4] + }) + + cur.close() + conn.close() + + self.log.info("Extracted a total of %d calls", len(self.results)) diff --git a/mvt/ios/modules/fs/chrome_favicon.py b/mvt/ios/modules/fs/chrome_favicon.py new file mode 100644 index 0000000..666e7c8 --- /dev/null +++ b/mvt/ios/modules/fs/chrome_favicon.py @@ -0,0 +1,78 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021 MVT Project Developers. +# See the file 'LICENSE' for usage and copying permissions, or find a copy at +# https://github.com/mvt-project/mvt/blob/main/LICENSE + +import sqlite3 + +from mvt.common.utils import convert_chrometime_to_unix, convert_timestamp_to_iso + +from .base import IOSExtraction + +CHROME_FAVICON_BACKUP_IDS = [ + "55680ab883d0fdcffd94f959b1632e5fbbb18c5b" +] + +# TODO: Confirm Chrome database path. +CHROME_FAVICON_ROOT_PATHS = [ + "private/var/mobile/Containers/Data/Application/*/Library/Application Support/Google/Chrome/Default/Favicons", +] + +class ChromeFavicon(IOSExtraction): + """This module extracts all Chrome favicon records.""" + + def __init__(self, file_path=None, base_folder=None, output_folder=None, + fast_mode=False, log=None, results=[]): + super().__init__(file_path=file_path, base_folder=base_folder, + output_folder=output_folder, fast_mode=fast_mode, + log=log, results=results) + + def serialize(self, record): + return { + "timestamp": record["isodate"], + "module": self.__class__.__name__, + "event": "new_favicon", + "data": f"{record['icon_url']} from {record['url']}" + } + + def check_indicators(self): + if not self.indicators: + return + + for result in self.results: + if self.indicators.check_domain(result["url"]) or self.indicators.check_domain(result["icon_url"]): + self.detected.append(result) + + def run(self): + self._find_ios_database(backup_ids=CHROME_FAVICON_BACKUP_IDS, root_paths=CHROME_FAVICON_ROOT_PATHS) + self.log.info("Found Chrome favicon cache database at path: %s", self.file_path) + + conn = sqlite3.connect(self.file_path) + + # Fetch icon cache + cur = conn.cursor() + cur.execute("""SELECT + icon_mapping.page_url, + favicons.url, + favicon_bitmaps.last_updated, + favicon_bitmaps.last_requested + FROM icon_mapping + JOIN favicon_bitmaps ON icon_mapping.icon_id = favicon_bitmaps.icon_id + JOIN favicons ON icon_mapping.icon_id = favicons.id + ORDER BY icon_mapping.id;""") + + items = [] + for item in cur: + last_timestamp = int(item[2]) or int(item[3]) + items.append(dict( + url=item[0], + icon_url=item[1], + timestamp=last_timestamp, + isodate=convert_timestamp_to_iso(convert_chrometime_to_unix(last_timestamp)), + )) + + cur.close() + conn.close() + + self.log.info("Extracted a total of %d favicon records", len(items)) + self.results = sorted(items, key=lambda item: item["isodate"]) diff --git a/mvt/ios/modules/fs/chrome_history.py b/mvt/ios/modules/fs/chrome_history.py new file mode 100644 index 0000000..8d8a64e --- /dev/null +++ b/mvt/ios/modules/fs/chrome_history.py @@ -0,0 +1,69 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021 MVT Project Developers. +# See the file 'LICENSE' for usage and copying permissions, or find a copy at +# https://github.com/mvt-project/mvt/blob/main/LICENSE + +import sqlite3 + +from mvt.common.utils import convert_chrometime_to_unix, convert_timestamp_to_iso + +from .base import IOSExtraction + +CHROME_HISTORY_BACKUP_IDS = [ + "faf971ce92c3ac508c018dce1bef2a8b8e9838f1", +] + +# TODO: Confirm Chrome database path. +CHROME_HISTORY_ROOT_PATHS = [ + "private/var/mobile/Containers/Data/Application/*/Library/Application Support/Google/Chrome/Default/History", +] + +class ChromeHistory(IOSExtraction): + """This module extracts all Chome visits.""" + + def __init__(self, file_path=None, base_folder=None, output_folder=None, + fast_mode=False, log=None, results=[]): + super().__init__(file_path=file_path, base_folder=base_folder, + output_folder=output_folder, fast_mode=fast_mode, + log=log, results=results) + + def serialize(self, record): + return { + "timestamp": record["isodate"], + "module": self.__class__.__name__, + "event": "visit", + "data": f"{record['id']} - {record['url']} (visit ID: {record['visit_id']}, redirect source: {record['redirect_source']})" + } + + def run(self): + self._find_ios_database(backup_ids=CHROME_HISTORY_BACKUP_IDS, root_paths=CHROME_HISTORY_ROOT_PATHS) + self.log.info("Found Chrome history database at path: %s", self.file_path) + + conn = sqlite3.connect(self.file_path) + cur = conn.cursor() + cur.execute(""" + SELECT + urls.id, + urls.url, + visits.id, + visits.visit_time, + visits.from_visit + FROM urls + JOIN visits ON visits.url = urls.id + ORDER BY visits.visit_time; + """) + + for item in cur: + self.results.append(dict( + id=item[0], + url=item[1], + visit_id=item[2], + timestamp=item[3], + isodate=convert_timestamp_to_iso(convert_chrometime_to_unix(item[3])), + redirect_source=item[4], + )) + + cur.close() + conn.close() + + self.log.info("Extracted a total of %d history items", len(self.results)) diff --git a/mvt/ios/modules/fs/contacts.py b/mvt/ios/modules/fs/contacts.py new file mode 100644 index 0000000..ac99842 --- /dev/null +++ b/mvt/ios/modules/fs/contacts.py @@ -0,0 +1,52 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021 MVT Project Developers. +# See the file 'LICENSE' for usage and copying permissions, or find a copy at +# https://github.com/mvt-project/mvt/blob/main/LICENSE + +import sqlite3 + +from .base import IOSExtraction + +CONTACTS_BACKUP_IDS = [ + "31bb7ba8914766d4ba40d6dfb6113c8b614be442", +] +CONTACTS_ROOT_PATHS = [ + "private/var/mobile/Library/AddressBook/AddressBook.sqlitedb", +] + +class Contacts(IOSExtraction): + """This module extracts all contact details from the phone's address book.""" + + def __init__(self, file_path=None, base_folder=None, output_folder=None, + fast_mode=False, log=None, results=[]): + super().__init__(file_path=file_path, base_folder=base_folder, + output_folder=output_folder, fast_mode=fast_mode, + log=log, results=results) + + def run(self): + self._find_ios_database(backup_ids=CONTACTS_BACKUP_IDS, root_paths=CONTACTS_ROOT_PATHS) + self.log.info("Found Contacts database at path: %s", self.file_path) + + conn = sqlite3.connect(self.file_path) + cur = conn.cursor() + cur.execute(""" + SELECT + multi.value, person.first, person.middle, person.last, + person.organization + FROM ABPerson person, ABMultiValue multi + WHERE person.rowid = multi.record_id and multi.value not null + ORDER by person.rowid ASC; + """) + names = [description[0] for description in cur.description] + + for entry in cur: + new_contact = dict() + for index, value in enumerate(entry): + new_contact[names[index]] = value + + self.results.append(new_contact) + + cur.close() + conn.close() + + self.log.info("Extracted a total of %d contacts from the address book", len(self.results)) diff --git a/mvt/ios/modules/fs/filesystem.py b/mvt/ios/modules/fs/filesystem.py new file mode 100644 index 0000000..fc170f6 --- /dev/null +++ b/mvt/ios/modules/fs/filesystem.py @@ -0,0 +1,51 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021 MVT Project Developers. +# See the file 'LICENSE' for usage and copying permissions, or find a copy at +# https://github.com/mvt-project/mvt/blob/main/LICENSE + +import os +import datetime + +from mvt.common.utils import convert_timestamp_to_iso + +from .base import IOSExtraction + +class Filesystem(IOSExtraction): + """This module extracts creation and modification date of files from a + full file-system dump.""" + + def __init__(self, file_path=None, base_folder=None, output_folder=None, + fast_mode=False, log=None, results=[]): + super().__init__(file_path=file_path, base_folder=base_folder, + output_folder=output_folder, fast_mode=fast_mode, + log=log, results=results) + + def serialize(self, record): + return { + "timestamp": record["modified"], + "module": self.__class__.__name__, + "event": f"file_modified", + "data": record["file_path"], + } + + def check_indicators(self): + if not self.indicators: + return + + for result in self.results: + if self.indicators.check_file(result["file_path"]): + self.detected.append(result) + + def run(self): + for root, dirs, files in os.walk(self.base_folder): + for file_name in files: + try: + file_path = os.path.join(root, file_name) + result = { + "file_path": os.path.relpath(file_path, self.base_folder), + "modified": convert_timestamp_to_iso(datetime.datetime.utcfromtimestamp(os.stat(file_path).st_mtime)), + } + except: + continue + else: + self.results.append(result) diff --git a/mvt/ios/modules/fs/firefox_favicon.py b/mvt/ios/modules/fs/firefox_favicon.py new file mode 100644 index 0000000..ad0c09d --- /dev/null +++ b/mvt/ios/modules/fs/firefox_favicon.py @@ -0,0 +1,82 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021 MVT Project Developers. +# See the file 'LICENSE' for usage and copying permissions, or find a copy at +# https://github.com/mvt-project/mvt/blob/main/LICENSE + +import sqlite3 + +from datetime import datetime +from mvt.common.url import URL +from mvt.common.utils import convert_mactime_to_unix, convert_timestamp_to_iso + +from .base import IOSExtraction + +FIREFOX_HISTORY_BACKUP_IDS = [ + "2e57c396a35b0d1bcbc624725002d98bd61d142b", +] +FIREFOX_HISTORY_ROOT_PATHS = [ + "private/var/mobile/profile.profile/browser.db", +] + +class FirefoxFavicon(IOSExtraction): + """This module extracts all Firefox favicon""" + + def __init__(self, file_path=None, base_folder=None, output_folder=None, + fast_mode=False, log=None, results=[]): + super().__init__(file_path=file_path, base_folder=base_folder, + output_folder=output_folder, fast_mode=fast_mode, + log=log, results=results) + + def serialize(self, record): + return { + "timestamp": record["isodate"], + "module": self.__class__.__name__, + "event": "firefox_history", + "data": f"Firefox favicon {record['url']} when visiting {record['history_url']}", + } + + def check_indicators(self): + if not self.indicators: + return + + for result in self.results: + if self.indicators.check_domain(result["url"]) or self.indicators.check_domain(result["history_url"]): + self.detected.append(result) + + def run(self): + self._find_ios_database(backup_ids=FIREFOX_HISTORY_BACKUP_IDS, root_paths=FIREFOX_HISTORY_ROOT_PATHS) + self.log.info("Found Firefox favicon database at path: %s", self.file_path) + + conn = sqlite3.connect(self.file_path) + cur = conn.cursor() + cur.execute(""" + SELECT + favicons.id, + favicons.url, + favicons.width, + favicons.height, + favicons.type, + favicons.date, + history.id, + history.url + FROM favicons + INNER JOIN favicon_sites ON favicon_sites.faviconID = favicons.id + INNER JOIN history ON favicon_sites.siteID = history.id; + """) + + for item in cur: + self.results.append(dict( + id=item[0], + url=item[1], + width=item[2], + height=item[3], + type=item[4], + isodate=convert_timestamp_to_iso(datetime.utcfromtimestamp(item[5])), + history_id=item[6], + history_url=item[7] + )) + + cur.close() + conn.close() + + self.log.info("Extracted a total of %d history items", len(self.results)) diff --git a/mvt/ios/modules/fs/firefox_history.py b/mvt/ios/modules/fs/firefox_history.py new file mode 100644 index 0000000..8d9bff9 --- /dev/null +++ b/mvt/ios/modules/fs/firefox_history.py @@ -0,0 +1,78 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021 MVT Project Developers. +# See the file 'LICENSE' for usage and copying permissions, or find a copy at +# https://github.com/mvt-project/mvt/blob/main/LICENSE + +import sqlite3 + +from datetime import datetime +from mvt.common.url import URL +from mvt.common.utils import convert_mactime_to_unix, convert_timestamp_to_iso + +from .base import IOSExtraction + +FIREFOX_HISTORY_BACKUP_IDS = [ + "2e57c396a35b0d1bcbc624725002d98bd61d142b", +] +FIREFOX_HISTORY_ROOT_PATHS = [ + "private/var/mobile/profile.profile/browser.db", +] + +class FirefoxHistory(IOSExtraction): + """This module extracts all Firefox visits and tries to detect potential + network injection attacks.""" + + def __init__(self, file_path=None, base_folder=None, output_folder=None, + fast_mode=False, log=None, results=[]): + super().__init__(file_path=file_path, base_folder=base_folder, + output_folder=output_folder, fast_mode=fast_mode, + log=log, results=results) + + def serialize(self, record): + return { + "timestamp": record["isodate"], + "module": self.__class__.__name__, + "event": "firefox_history", + "data": f"Firefox visit with ID {record['id']} to URL: {record['url']}", + } + + def check_indicators(self): + if not self.indicators: + return + + for result in self.results: + if self.indicators.check_domain(result["url"]): + self.detected.append(result) + + def run(self): + self._find_ios_database(backup_ids=FIREFOX_HISTORY_BACKUP_IDS, root_paths=FIREFOX_HISTORY_ROOT_PATHS) + self.log.info("Found Firefox history database at path: %s", self.file_path) + + conn = sqlite3.connect(self.file_path) + cur = conn.cursor() + cur.execute(""" + SELECT + visits.id, + visits.date/1000000, + history.url, + history.title, + visits.is_local, + visits.type + FROM visits, history + WHERE visits.siteID = history.id; + """) + + for item in cur: + self.results.append(dict( + id=item[0], + isodate=convert_timestamp_to_iso(datetime.utcfromtimestamp(item[1])), + url=item[2], + title=item[3], + i1000000s_local=item[4], + type=item[5] + )) + + cur.close() + conn.close() + + self.log.info("Extracted a total of %d history items", len(self.results)) diff --git a/mvt/ios/modules/fs/idstatuscache.py b/mvt/ios/modules/fs/idstatuscache.py new file mode 100644 index 0000000..b447e58 --- /dev/null +++ b/mvt/ios/modules/fs/idstatuscache.py @@ -0,0 +1,86 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021 MVT Project Developers. +# See the file 'LICENSE' for usage and copying permissions, or find a copy at +# https://github.com/mvt-project/mvt/blob/main/LICENSE + +import os +import glob +import biplist +import collections + +from mvt.common.utils import convert_mactime_to_unix, convert_timestamp_to_iso + +from .base import IOSExtraction + +IDSTATUSCACHE_BACKUP_IDS = [ + "6b97989189901ceaa4e5be9b7f05fb584120e27b", +] +IDSTATUSCACHE_ROOT_PATHS = [ + "private/var/mobile/Library/Preferences/com.apple.identityservices.idstatuscache.plist", +] + +class IDStatusCache(IOSExtraction): + """Extracts Apple Authentication information from idstatuscache.plist""" + + def __init__(self, file_path=None, base_folder=None, output_folder=None, + fast_mode=False, log=None, results=[]): + super().__init__(file_path=file_path, base_folder=base_folder, + output_folder=output_folder, fast_mode=fast_mode, + log=log, results=results) + + def serialize(self, record): + return { + "timestamp": record["isodate"], + "module": self.__class__.__name__, + "event": "lookup", + "data": f"Lookup of {record['user']} within {record['package']} (Status {record['idstatus']})" + } + + def check_indicators(self): + if not self.indicators: + return + + for result in self.results: + if result["user"].startswith("mailto:"): + email = result["user"][7:].strip("'") + if self.indicators.check_email(email): + self.detected.append(result) + continue + + if "\\x00\\x00" in result["user"]: + self.log.warning("Found an ID Status Cache entry with suspicious patterns: %s", + result["user"]) + self.detected.append(result) + + def run(self): + self._find_ios_database(backup_ids=IDSTATUSCACHE_BACKUP_IDS, root_paths=IDSTATUSCACHE_ROOT_PATHS) + self.log.info("Found IDStatusCache plist at path: %s", self.file_path) + + file_plist = biplist.readPlist(self.file_path) + + id_status_cache_entries = [] + for app in file_plist: + if not isinstance(file_plist[app], dict): + continue + + for entry in file_plist[app]: + try: + lookup_date = file_plist[app][entry]["LookupDate"] + id_status = file_plist[app][entry]["IDStatus"] + except KeyError: + continue + + id_status_cache_entries.append({ + "package": app, + "user": entry.replace("\x00", "\\x00"), + "isodate": convert_timestamp_to_iso(convert_mactime_to_unix(lookup_date)), + "idstatus": id_status, + }) + + entry_counter = collections.Counter([entry["user"] for entry in id_status_cache_entries]) + for entry in id_status_cache_entries: + # Add total count of occurrences to the status cache entry + entry["occurrences"] = entry_counter[entry["user"]] + self.results.append(entry) + + self.log.info("Extracted a total of %d ID Status Cache entries", len(self.results)) diff --git a/mvt/ios/modules/fs/interactionc.py b/mvt/ios/modules/fs/interactionc.py new file mode 100644 index 0000000..a685a93 --- /dev/null +++ b/mvt/ios/modules/fs/interactionc.py @@ -0,0 +1,174 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021 MVT Project Developers. +# See the file 'LICENSE' for usage and copying permissions, or find a copy at +# https://github.com/mvt-project/mvt/blob/main/LICENSE + +import sqlite3 +from base64 import b64encode + +from mvt.common.utils import convert_mactime_to_unix, convert_timestamp_to_iso + +from .base import IOSExtraction + +INTERACTIONC_BACKUP_IDS = [ + "1f5a521220a3ad80ebfdc196978df8e7a2e49dee", +] +INTERACTIONC_ROOT_PATHS = [ + "private/var/mobile/Library/CoreDuet/People/interactionC.db", +] + +class InteractionC(IOSExtraction): + """This module extracts data from InteractionC db.""" + + def __init__(self, file_path=None, base_folder=None, output_folder=None, + fast_mode=False, log=None, results=[]): + super().__init__(file_path=file_path, base_folder=base_folder, + output_folder=output_folder, fast_mode=fast_mode, + log=log, results=results) + + self.timestamps = [ + "start_date", + "end_date", + "interactions_creation_date", + "contacts_creation_date", + "first_incoming_recipient_date", + "first_incoming_sender_date", + "first_outgoing_recipient_date", + "last_incoming_sender_date", + "last_incoming_recipient_date", + "last_outgoing_recipient_date", + ] + + def serialize(self, record): + records = [] + processed = [] + for ts in self.timestamps: + # Check if the record has the current timestamp. + if ts not in record or not record[ts]: + continue + + # Check if the timestamp was already processed. + if record[ts] in processed: + continue + + records.append({ + "timestamp": record[ts], + "module": self.__class__.__name__, + "event": ts, + "data": f"[{record['bundle_id']}] {record['account']} - from {record['sender_display_name']} " \ + f"({record['sender_identifier']}) to {record['recipient_display_name']} " \ + f"({record['recipient_identifier']}): {record['content']}" + }) + processed.append(record[ts]) + + return records + + def run(self): + self._find_ios_database(backup_ids=INTERACTIONC_BACKUP_IDS, root_paths=INTERACTIONC_ROOT_PATHS) + self.log.info("Found InteractionC database at path: %s", self.file_path) + + conn = sqlite3.connect(self.file_path) + cur = conn.cursor() + + # TODO: Support all versions. + # Taken from: + # https://github.com/mac4n6/APOLLO/blob/master/modules/interaction_contact_interactions.txt + cur.execute(""" + SELECT + ZINTERACTIONS.ZSTARTDATE, + ZINTERACTIONS.ZENDDATE, + ZINTERACTIONS.ZBUNDLEID, + ZINTERACTIONS.ZACCOUNT, + ZINTERACTIONS.ZTARGETBUNDLEID, + CASE ZINTERACTIONS.ZDIRECTION + WHEN '0' THEN 'INCOMING' + WHEN '1' THEN 'OUTGOING' + END 'DIRECTION', + ZCONTACTS.ZDISPLAYNAME, + ZCONTACTS.ZIDENTIFIER, + ZCONTACTS.ZPERSONID, + RECEIPIENTCONACT.ZDISPLAYNAME, + RECEIPIENTCONACT.ZIDENTIFIER, + RECEIPIENTCONACT.ZPERSONID, + ZINTERACTIONS.ZRECIPIENTCOUNT, + ZINTERACTIONS.ZDOMAINIDENTIFIER, + ZINTERACTIONS.ZISRESPONSE, + ZATTACHMENT.ZCONTENTTEXT, + ZATTACHMENT.ZUTI, + ZATTACHMENT.ZCONTENTURL, + ZATTACHMENT.ZSIZEINBYTES, + ZATTACHMENT.ZPHOTOLOCALIDENTIFIER, + HEX(ZATTACHMENT.ZIDENTIFIER), + ZATTACHMENT.ZCLOUDIDENTIFIER, + ZCONTACTS.ZINCOMINGRECIPIENTCOUNT, + ZCONTACTS.ZINCOMINGSENDERCOUNT, + ZCONTACTS.ZOUTGOINGRECIPIENTCOUNT, + ZINTERACTIONS.ZCREATIONDATE, + ZCONTACTS.ZCREATIONDATE, + ZCONTACTS.ZFIRSTINCOMINGRECIPIENTDATE, + ZCONTACTS.ZFIRSTINCOMINGSENDERDATE, + ZCONTACTS.ZFIRSTOUTGOINGRECIPIENTDATE, + ZCONTACTS.ZLASTINCOMINGSENDERDATE, + ZCONTACTS.ZLASTINCOMINGRECIPIENTDATE, + ZCONTACTS.ZLASTOUTGOINGRECIPIENTDATE, + ZCONTACTS.ZCUSTOMIDENTIFIER, + ZINTERACTIONS.ZCONTENTURL, + ZINTERACTIONS.ZLOCATIONUUID, + ZINTERACTIONS.ZGROUPNAME, + ZINTERACTIONS.ZDERIVEDINTENTIDENTIFIER, + ZINTERACTIONS.Z_PK + FROM ZINTERACTIONS + LEFT JOIN ZCONTACTS ON ZINTERACTIONS.ZSENDER = ZCONTACTS.Z_PK + LEFT JOIN Z_1INTERACTIONS ON ZINTERACTIONS.Z_PK == Z_1INTERACTIONS.Z_3INTERACTIONS + LEFT JOIN ZATTACHMENT ON Z_1INTERACTIONS.Z_1ATTACHMENTS == ZATTACHMENT.Z_PK + LEFT JOIN Z_2INTERACTIONRECIPIENT ON ZINTERACTIONS.Z_PK== Z_2INTERACTIONRECIPIENT.Z_3INTERACTIONRECIPIENT + LEFT JOIN ZCONTACTS RECEIPIENTCONACT ON Z_2INTERACTIONRECIPIENT.Z_2RECIPIENTS== RECEIPIENTCONACT.Z_PK; + """) + names = [description[0] for description in cur.description] + + for item in cur: + self.results.append({ + "start_date": convert_timestamp_to_iso(convert_mactime_to_unix(item[0])), + "end_date": convert_timestamp_to_iso(convert_mactime_to_unix(item[1])), + "bundle_id": item[2], + "account": item[3], + "target_bundle_id": item[4], + "direction": item[5], + "sender_display_name": item[6], + "sender_identifier": item[7], + "sender_personid": item[8], + "recipient_display_name": item[9], + "recipient_identifier": item[10], + "recipient_personid": item[11], + "recipient_count": item[12], + "domain_identifier": item[13], + "is_response": item[14], + "content": item[15], + "uti": item[16], + "content_url": item[17], + "size": item[18], + "photo_local_id": item[19], + "attachment_id": item[20], + "cloud_id": item[21], + "incoming_recipient_count": item[22], + "incoming_sender_count": item[23], + "outgoing_recipient_count": item[24], + "interactions_creation_date": convert_timestamp_to_iso(convert_mactime_to_unix(item[25])) if item[25] else None, + "contacts_creation_date": convert_timestamp_to_iso(convert_mactime_to_unix(item[26])) if item[26] else None, + "first_incoming_recipient_date": convert_timestamp_to_iso(convert_mactime_to_unix(item[27])) if item[27] else None, + "first_incoming_sender_date": convert_timestamp_to_iso(convert_mactime_to_unix(item[28])) if item[28] else None, + "first_outgoing_recipient_date": convert_timestamp_to_iso(convert_mactime_to_unix(item[29])) if item[29] else None, + "last_incoming_sender_date": convert_timestamp_to_iso(convert_mactime_to_unix(item[30])) if item[30] else None, + "last_incoming_recipient_date": convert_timestamp_to_iso(convert_mactime_to_unix(item[31])) if item[31] else None, + "last_outgoing_recipient_date": convert_timestamp_to_iso(convert_mactime_to_unix(item[32])) if item[32] else None, + "custom_id": item[33], + "location_uuid": item[35], + "group_name": item[36], + "derivied_intent_id": item[37], + "table_id": item[38] + }) + + cur.close() + conn.close() + + self.log.info("Extracted a total of %d InteractionC events", len(self.results)) diff --git a/mvt/ios/modules/fs/locationd.py b/mvt/ios/modules/fs/locationd.py new file mode 100644 index 0000000..fe39f17 --- /dev/null +++ b/mvt/ios/modules/fs/locationd.py @@ -0,0 +1,69 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021 MVT Project Developers. +# See the file 'LICENSE' for usage and copying permissions, or find a copy at +# https://github.com/mvt-project/mvt/blob/main/LICENSE + +import os +import glob +import biplist + +from mvt.common.utils import convert_mactime_to_unix, convert_timestamp_to_iso + +from .base import IOSExtraction + +LOCATIOND_BACKUP_IDS = [ + "a690d7769cce8904ca2b67320b107c8fe5f79412", +] +LOCATIOND_ROOT_PATHS = [ + "private/var/mobile/Library/Caches/locationd/clients.plist", +] + +class LocationdClients(IOSExtraction): + """Extract information from apps who used geolocation""" + + def __init__(self, file_path=None, base_folder=None, output_folder=None, + fast_mode=False, log=None, results=[]): + super().__init__(file_path=file_path, base_folder=base_folder, + output_folder=output_folder, fast_mode=fast_mode, + log=log, results=results) + self.timestamps = [ + "ConsumptionPeriodBegin", + "ReceivingLocationInformationTimeStopped", + "VisitTimeStopped", + "LocationTimeStopped", + "BackgroundLocationTimeStopped", + "SignificantTimeStopped", + "NonPersistentSignificantTimeStopped", + "FenceTimeStopped", + "BeaconRegionTimeStopped", + ] + + def serialize(self, record): + records = [] + for ts in self.timestamps: + if ts in record.keys(): + records.append({ + "timestamp": entry[ts], + "module": self.__class__.__name__, + "event": ts, + "data": f"{ts} from {entry['package']}" + }) + + return records + + def run(self): + self._find_ios_database(backup_ids=LOCATIOND_BACKUP_IDS, root_paths=LOCATIOND_ROOT_PATHS) + self.log.info("Found Locationd Clients plist at path: %s", self.file_path) + file_plist = biplist.readPlist(self.file_path) + + for app in file_plist: + if file_plist[app] is dict: + result = file_plist[app] + result["package"] = app + for ts in self.timestamps: + if ts in result.keys(): + result[ts] = convert_timestamp_to_iso(convert_mactime_to_unix(result[date])) + + self.results.append(result) + + self.log.info("Extracted a total of %d Locationd Clients entries", len(self.results)) diff --git a/mvt/ios/modules/fs/manifest.py b/mvt/ios/modules/fs/manifest.py new file mode 100644 index 0000000..d5c1c74 --- /dev/null +++ b/mvt/ios/modules/fs/manifest.py @@ -0,0 +1,131 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021 MVT Project Developers. +# See the file 'LICENSE' for usage and copying permissions, or find a copy at +# https://github.com/mvt-project/mvt/blob/main/LICENSE + +import io +import os +import biplist +import sqlite3 +import datetime + +from mvt.common.utils import convert_timestamp_to_iso + +from .base import IOSExtraction + +class Manifest(IOSExtraction): + """This module extracts information from a backup Manifest.db file.""" + + def __init__(self, file_path=None, base_folder=None, output_folder=None, + fast_mode=False, log=None, results=[]): + super().__init__(file_path=file_path, base_folder=base_folder, + output_folder=output_folder, fast_mode=fast_mode, + log=log, results=results) + + def _get_key(self, dictionary, key): + """ + Unserialized plist objects can have keys which are str or byte types + + This is a helper to try fetch a key as both a byte or string type. + """ + return dictionary.get(key.encode("utf-8"), None) or dictionary.get(key, None) + + def _convert_timestamp(self, timestamp_or_unix_time_int): + """Older iOS versions stored the manifest times as unix timestamps.""" + if isinstance(timestamp_or_unix_time_int, datetime.datetime): + return convert_timestamp_to_iso(timestamp_or_unix_time_int) + else: + timestamp = datetime.datetime.utcfromtimestamp(timestamp_or_unix_time_int) + return convert_timestamp_to_iso(timestamp) + + def serialize(self, record): + records = [] + for ts in set([record["created"], record["modified"], record["statusChanged"]]): + macb = "" + macb += "M" if ts == record["modified"] else "-" + macb += "-" + macb += "C" if ts == record["statusChanged"] else "-" + macb += "B" if ts == record["created"] else "-" + + records.append({ + "timestamp": ts, + "module": self.__class__.__name__, + "event": macb, + "data": f"{record['relativePath']} - {record['domain']}" + }) + + return records + + def check_indicators(self): + if not self.indicators: + return + + for result in self.results: + if not "relativePath" in result: + continue + + if os.path.basename(result["relativePath"]) == "com.apple.CrashReporter.plist" and result["domain"] == "RootDomain": + self.log.warning("Found a potentially suspicious \"com.apple.CrashReporter.plist\" file created in RootDomain") + self.detected.append(result) + continue + + if self.indicators.check_file(result["relativePath"]): + self.log.warning("Found a known malicious file at path: %s", result["relativePath"]) + self.detected.append(result) + continue + + relPath = result["relativePath"].lower() + for ioc in self.indicators.ioc_domains: + if ioc.lower() in relPath: + self.log.warning("Found mention of domain \"%s\" in a backup file with path: %s", + ioc, relPath) + self.detected.append(result) + + def run(self): + manifest_db_path = os.path.join(self.base_folder, "Manifest.db") + if not os.path.isfile(manifest_db_path): + raise FileNotFoundError("Impossible to find the module's database file") + + self.log.info("Found Manifest.db database at path: %s", manifest_db_path) + + conn = sqlite3.connect(manifest_db_path) + cur = conn.cursor() + + cur.execute("SELECT * FROM Files;") + names = [description[0] for description in cur.description] + + for file_entry in cur: + file_data = dict() + for index, value in enumerate(file_entry): + file_data[names[index]] = value + + cleaned_metadata = { + "fileID": file_data["fileID"], + "domain": file_data["domain"], + "relativePath": file_data["relativePath"], + "flags": file_data["flags"], + "created": "", + } + + if file_data["file"]: + try: + file_plist = biplist.readPlist(io.BytesIO(file_data["file"])) + file_metadata = self._get_key(file_plist, "$objects")[1] + cleaned_metadata.update({ + "created": self._convert_timestamp(self._get_key(file_metadata, "Birth")), + "modified": self._convert_timestamp(self._get_key(file_metadata, "LastModified")), + "statusChanged": self._convert_timestamp(self._get_key(file_metadata, "LastStatusChange")), + "mode": oct(self._get_key(file_metadata, "Mode")), + "owner": self._get_key(file_metadata, "UserID"), + "size": self._get_key(file_metadata, "Size"), + }) + except: + self.log.exception("Error reading manifest file metadata") + pass + + self.results.append(cleaned_metadata) + + cur.close() + conn.close() + + self.log.info("Extracted a total of %d file metadata items", len(self.results)) diff --git a/mvt/ios/modules/fs/net_base.py b/mvt/ios/modules/fs/net_base.py new file mode 100644 index 0000000..b5cd550 --- /dev/null +++ b/mvt/ios/modules/fs/net_base.py @@ -0,0 +1,224 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021 MVT Project Developers. +# See the file 'LICENSE' for usage and copying permissions, or find a copy at +# https://github.com/mvt-project/mvt/blob/main/LICENSE + +import sqlite3 +import operator +from pathlib import Path + +from mvt.common.utils import convert_mactime_to_unix, convert_timestamp_to_iso + +from .base import IOSExtraction + +class NetBase(IOSExtraction): + """This class provides a base for DataUsage and NetUsage extraction modules.""" + + def _extract_net_data(self): + conn = sqlite3.connect(self.file_path) + cur = conn.cursor() + cur.execute("""SELECT + ZPROCESS.ZFIRSTTIMESTAMP, + ZPROCESS.ZTIMESTAMP, + ZPROCESS.ZPROCNAME, + ZPROCESS.ZBUNDLENAME, + ZPROCESS.Z_PK, + ZLIVEUSAGE.ZWIFIIN, + ZLIVEUSAGE.ZWIFIOUT, + ZLIVEUSAGE.ZWWANIN, + ZLIVEUSAGE.ZWWANOUT, + ZLIVEUSAGE.Z_PK, + ZLIVEUSAGE.ZHASPROCESS, + ZLIVEUSAGE.ZTIMESTAMP + FROM ZLIVEUSAGE + LEFT JOIN ZPROCESS ON ZLIVEUSAGE.ZHASPROCESS = ZPROCESS.Z_PK;""") + + items = [] + for item in cur: + # ZPROCESS records can be missing after the JOIN. Handle NULL timestamps. + if item[0] and item[1]: + first_isodate = convert_timestamp_to_iso(convert_mactime_to_unix(item[0])) + isodate = convert_timestamp_to_iso(convert_mactime_to_unix(item[1])) + else: + first_isodate = item[0] + isodate = item[1] + + if item[11]: + live_timestamp = convert_timestamp_to_iso(convert_mactime_to_unix(item[11])) + else: + live_timestamp = "" + + items.append(dict( + first_isodate=first_isodate, + isodate=isodate, + proc_name=item[2], + bundle_id=item[3], + proc_id=item[4], + wifi_in=item[5], + wifi_out=item[6], + wwan_in=item[7], + wwan_out=item[8], + live_id=item[9], + live_proc_id=item[10], + live_isodate=live_timestamp, + )) + + cur.close() + conn.close() + + self.log.info("Extracted information on %d processes", len(items)) + self.results = items + + def serialize(self, record): + record_data = f"{record['proc_name']} (Bundle ID: {record['bundle_id']}, ID: {record['proc_id']})" + record_data_usage = record_data + f" WIFI IN: {record['wifi_in']}, WIFI OUT: {record['wifi_out']} - " \ + f"WWAN IN: {record['wwan_in']}, WWAN OUT: {record['wwan_out']}" + + records = [{ + "timestamp": record["live_isodate"], + "module": self.__class__.__name__, + "event": "live_usage", + "data": record_data_usage, + }] + + # Only included first_usage and current_usage records when a ZPROCESS entry exists. + if "MANIPULATED" not in record["proc_name"] and "MISSING" not in record["proc_name"]: + records.extend([ + { + "timestamp": record["first_isodate"], + "module": self.__class__.__name__, + "event": "first_usage", + "data": record_data, + }, + { + "timestamp": record["isodate"], + "module": self.__class__.__name__, + "event": "current_usage", + "data": record_data, + } + ]) + return records + + def _find_suspicious_processes(self): + if not self.is_fs_dump: + return + + if not self.results: + return + + # If we are instructed to run fast, we skip this. + if self.fast_mode: + self.log.info("Flag --fast was enabled: skipping extended search for suspicious processes") + return + + self.log.info("Extended search for suspicious processes ...") + + files = [] + for posix_path in Path(self.base_folder).rglob("*"): + try: + if not posix_path.is_file(): + continue + except PermissionError: + continue + + files.append([posix_path.name, posix_path.__str__()]) + + for proc in self.results: + if not proc["bundle_id"]: + self.log.debug("Found process with no Bundle ID with name: %s", proc["proc_name"]) + + binary_path = None + for file in files: + if proc["proc_name"] == file[0]: + binary_path = file[1] + break + + if binary_path: + self.log.debug("Located at %s", binary_path) + else: + msg = f"Could not find the binary associated with the process with name {proc['proc_name']}" + if len(proc["proc_name"]) == 16: + msg = msg + " (However, the process name might have been truncated in the database)" + + self.log.warning(msg) + + def check_manipulated(self): + """Check for missing or manipulate DB entries + """ + # Don't show duplicates for each missing process. + missing_process_cache = set() + for result in sorted(self.results, key=operator.itemgetter("live_isodate")): + if result["proc_id"]: + continue + + # Avoid duplicate warnings for same process. + if result["live_proc_id"] not in missing_process_cache: + missing_process_cache.add(result["live_proc_id"]) + self.log.warning("Found manipulated process entry %s. Entry on %s", + result["live_proc_id"], result["live_isodate"]) + + # Set manipulated proc timestamp so it appears in timeline. + result["first_isodate"] = result["isodate"] = result["live_isodate"] + result["proc_name"] = "MANIPULATED [process record deleted]" + self.detected.append(result) + + def find_deleted(self): + """Identify process which may have been deleted from the DataUsage database""" + results_by_proc = {proc["proc_id"]: proc for proc in self.results if proc["proc_id"]} + all_proc_id = sorted(results_by_proc.keys()) + + missing_procs, last_proc_id = {}, None + for proc_id in range(min(all_proc_id), max(all_proc_id)): + if proc_id not in all_proc_id: + previous_proc = results_by_proc[last_proc_id] + self.log.info("Missing process %d. Previous process at \"%s\" (%s)", + proc_id, previous_proc["first_isodate"], previous_proc["proc_name"]) + + missing_procs[proc_id] = { + "proc_id": proc_id, + "prev_proc_id": last_proc_id, + "prev_proc_name": previous_proc["proc_name"], + "prev_proc_bundle": previous_proc["bundle_id"], + "prev_proc_first": previous_proc["first_isodate"], + } + else: + last_proc_id = proc_id + + # Add a placeholder entry for the missing processes. + for proc_id, proc in missing_procs.items(): + # Set default DataUsage keys. + result = {key: None for key in self.results[0].keys()} + result["first_isodate"] = result["isodate"] = result["live_isodate"] = proc["prev_proc_first"] + result["proc_name"] = "MISSING [follows {}]".format(proc["prev_proc_name"]) + result["proc_id"] = result["live_proc_id"] = proc["proc_id"] + result["bundle_id"] = None + + self.results.append(result) + + self.results = sorted(self.results, key=operator.itemgetter("first_isodate")) + + def check_indicators(self): + if not self.indicators: + return + + for result in self.results: + try: + proc_name = result["proc_name"] + except KeyError: + continue + + # Process ID may be empty if process records have been manipulated. + if not result["proc_id"]: + continue + + if self.indicators.check_process(proc_name): + self.detected.append(result) + + # Check for manipulated process records. + # TODO: Catching KeyError for live_isodate for retro-compatibility. + # This is not very good. + try: + self.check_manipulated() + self.find_deleted() + except KeyError: + pass diff --git a/mvt/ios/modules/fs/net_datausage.py b/mvt/ios/modules/fs/net_datausage.py new file mode 100644 index 0000000..dc9d319 --- /dev/null +++ b/mvt/ios/modules/fs/net_datausage.py @@ -0,0 +1,30 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021 MVT Project Developers. +# See the file 'LICENSE' for usage and copying permissions, or find a copy at +# https://github.com/mvt-project/mvt/blob/main/LICENSE + +from .net_base import NetBase + +DATAUSAGE_BACKUP_IDS = [ + "0d609c54856a9bb2d56729df1d68f2958a88426b", +] +DATAUSAGE_ROOT_PATHS = [ + "private/var/wireless/Library/Databases/DataUsage.sqlite", +] + +class Datausage(NetBase): + """This class extracts data from DataUsage.sqlite and attempts to identify + any suspicious processes if running on a full filesystem dump.""" + + def __init__(self, file_path=None, base_folder=None, output_folder=None, + fast_mode=False, log=None, results=[]): + super().__init__(file_path=file_path, base_folder=base_folder, + output_folder=output_folder, fast_mode=fast_mode, + log=log, results=results) + + def run(self): + self._find_ios_database(backup_ids=DATAUSAGE_BACKUP_IDS, root_paths=DATAUSAGE_ROOT_PATHS) + self.log.info("Found DataUsage database at path: %s", self.file_path) + + self._extract_net_data() + self._find_suspicious_processes() diff --git a/mvt/ios/modules/fs/net_netusage.py b/mvt/ios/modules/fs/net_netusage.py new file mode 100644 index 0000000..5f0e853 --- /dev/null +++ b/mvt/ios/modules/fs/net_netusage.py @@ -0,0 +1,28 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021 MVT Project Developers. +# See the file 'LICENSE' for usage and copying permissions, or find a copy at +# https://github.com/mvt-project/mvt/blob/main/LICENSE + +from .net_base import NetBase + +NETUSAGE_ROOT_PATHS = [ + "private/var/networkd/netusage.sqlite", + "private/var/networkd/db/netusage.sqlite" +] + +class Netusage(NetBase): + """This class extracts data from netusage.sqlite and attempts to identify + any suspicious processes if running on a full filesystem dump.""" + + def __init__(self, file_path=None, base_folder=None, output_folder=None, + fast_mode=False, log=None, results=[]): + super().__init__(file_path=file_path, base_folder=base_folder, + output_folder=output_folder, fast_mode=fast_mode, + log=log, results=results) + + def run(self): + self._find_ios_database(root_paths=NETUSAGE_ROOT_PATHS) + self.log.info("Found NetUsage database at path: %s", self.file_path) + + self._extract_net_data() + self._find_suspicious_processes() diff --git a/mvt/ios/modules/fs/safari_browserstate.py b/mvt/ios/modules/fs/safari_browserstate.py new file mode 100644 index 0000000..98d6bcb --- /dev/null +++ b/mvt/ios/modules/fs/safari_browserstate.py @@ -0,0 +1,103 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021 MVT Project Developers. +# See the file 'LICENSE' for usage and copying permissions, or find a copy at +# https://github.com/mvt-project/mvt/blob/main/LICENSE + +import io +import biplist +import sqlite3 + +from mvt.common.utils import convert_mactime_to_unix, convert_timestamp_to_iso +from mvt.common.utils import keys_bytes_to_string + +from .base import IOSExtraction + +SAFARI_BROWSER_STATE_BACKUP_IDS = [ + "3a47b0981ed7c10f3e2800aa66bac96a3b5db28e", +] +SAFARI_BROWSER_STATE_ROOT_PATHS = [ + "private/var/mobile/Library/Safari/BrowserState.db", + "private/var/mobile/Containers/Data/Application/*/Library/Safari/BrowserState.db", +] + +class SafariBrowserState(IOSExtraction): + """This module extracts all Safari browser state records.""" + + def __init__(self, file_path=None, base_folder=None, output_folder=None, + fast_mode=False, log=None, results=[]): + super().__init__(file_path=file_path, base_folder=base_folder, + output_folder=output_folder, fast_mode=fast_mode, + log=log, results=results) + + def serialize(self, record): + return { + "timestamp": record["last_viewed_timestamp"], + "module": self.__class__.__name__, + "event": "tab", + "data": f"{record['tab_title']} - {record['tab_url']}" + } + + def check_indicators(self): + if not self.indicators: + return + + for result in self.results: + if "tab_url" in result and self.indicators.check_domain(result["tab_url"]): + self.detected.append(result) + continue + + if not "session_data" in result: + continue + + for session_entry in result["session_data"]: + if "entry_url" in session_entry and self.indicators.check_domain(session_entry["entry_url"]): + self.detected.append(result) + + def run(self): + self._find_ios_database(backup_ids=SAFARI_BROWSER_STATE_BACKUP_IDS, + root_paths=SAFARI_BROWSER_STATE_ROOT_PATHS) + self.log.info("Found Safari browser state database at path: %s", self.file_path) + + conn = sqlite3.connect(self.file_path) + + # Fetch valid icon cache. + cur = conn.cursor() + cur.execute("""SELECT + tabs.title, + tabs.url, + tabs.user_visible_url, + tabs.last_viewed_time, + tab_sessions.session_data + FROM tabs + JOIN tab_sessions ON tabs.uuid = tab_sessions.tab_uuid + ORDER BY tabs.last_viewed_time;""") + + session_history_count = 0 + for item in cur: + session_entries = [] + + if item[4]: + # Skip a 4 byte header before the plist content. + session_plist = item[4][4:] + session_data = biplist.readPlist(io.BytesIO(session_plist)) + session_data = keys_bytes_to_string(session_data) + + if "SessionHistoryEntries" in session_data["SessionHistory"]: + for session_entry in session_data["SessionHistory"]["SessionHistoryEntries"]: + session_history_count += 1 + session_entries.append(dict( + entry_title=session_entry["SessionHistoryEntryOriginalURL"], + entry_url=session_entry["SessionHistoryEntryURL"], + data_length=len(session_entry["SessionHistoryEntryData"]) if "SessionHistoryEntryData" in session_entry else 0, + )) + + self.results.append(dict( + tab_title=item[0], + tab_url=item[1], + tab_visible_url=item[2], + last_viewed_timestamp=convert_timestamp_to_iso(convert_mactime_to_unix(item[3])), + session_data=session_entries, + )) + + self.log.info("Extracted a total of %d tab records and %d session history entries", + len(self.results), session_history_count) diff --git a/mvt/ios/modules/fs/safari_favicon.py b/mvt/ios/modules/fs/safari_favicon.py new file mode 100644 index 0000000..3bd5c36 --- /dev/null +++ b/mvt/ios/modules/fs/safari_favicon.py @@ -0,0 +1,88 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021 MVT Project Developers. +# See the file 'LICENSE' for usage and copying permissions, or find a copy at +# https://github.com/mvt-project/mvt/blob/main/LICENSE + +import sqlite3 + +from mvt.common.utils import convert_mactime_to_unix, convert_timestamp_to_iso + +from .base import IOSExtraction + +SAFARI_FAVICON_ROOT_PATHS = [ + "private/var/mobile/Library/Image Cache/Favicons/Favicons.db", + "private/var/mobile/Containers/Data/Application/*/Library/Image Cache/Favicons/Favicons.db", +] + +class SafariFavicon(IOSExtraction): + """This module extracts all Safari favicon records.""" + + def __init__(self, file_path=None, base_folder=None, output_folder=None, + fast_mode=False, log=None, results=[]): + super().__init__(file_path=file_path, base_folder=base_folder, + output_folder=output_folder, fast_mode=fast_mode, + log=log, results=results) + + def serialize(self, record): + return { + "timestamp": record["isodate"], + "module": self.__class__.__name__, + "event": "safari_favicon", + "data": f"Safari favicon from {record['url']} with icon URL {record['icon_url']} ({record['type']})", + } + + def check_indicators(self): + if not self.indicators: + return + + for result in self.results: + if self.indicators.check_domain(result["url"]) or self.indicators.check_domain(result["icon_url"]): + self.detected.append(result) + + def run(self): + self._find_ios_database(root_paths=SAFARI_FAVICON_ROOT_PATHS) + self.log.info("Found Safari favicon cache database at path: %s", self.file_path) + + conn = sqlite3.connect(self.file_path) + + # Fetch valid icon cache. + cur = conn.cursor() + cur.execute("""SELECT + page_url.url, + icon_info.url, + icon_info.timestamp + FROM page_url + JOIN icon_info ON page_url.uuid = icon_info.uuid + ORDER BY icon_info.timestamp;""") + + items = [] + for item in cur: + items.append(dict( + url=item[0], + icon_url=item[1], + timestamp=item[2], + isodate=convert_timestamp_to_iso(convert_mactime_to_unix(item[2])), + type="valid", + )) + + # Fetch icons from the rejected icons table. + cur.execute("""SELECT + page_url, + icon_url, + timestamp + FROM rejected_resources ORDER BY timestamp;""") + + for item in cur: + items.append(dict( + url=item[0], + icon_url=item[1], + timestamp=item[2], + isodate=convert_timestamp_to_iso(convert_mactime_to_unix(item[2])), + type="rejected", + )) + + cur.close() + conn.close() + + self.log.info("Extracted a total of %d favicon records", len(items)) + self.results = sorted(items, key=lambda item: item["isodate"]) diff --git a/mvt/ios/modules/fs/safari_history.py b/mvt/ios/modules/fs/safari_history.py new file mode 100644 index 0000000..4c498a9 --- /dev/null +++ b/mvt/ios/modules/fs/safari_history.py @@ -0,0 +1,119 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021 MVT Project Developers. +# See the file 'LICENSE' for usage and copying permissions, or find a copy at +# https://github.com/mvt-project/mvt/blob/main/LICENSE + +import sqlite3 + +from mvt.common.url import URL +from mvt.common.utils import convert_mactime_to_unix, convert_timestamp_to_iso + +from .base import IOSExtraction + +SAFARI_HISTORY_BACKUP_IDS = [ + "e74113c185fd8297e140cfcf9c99436c5cc06b57", + "1a0e7afc19d307da602ccdcece51af33afe92c53", +] +SAFARI_HISTORY_ROOT_PATHS = [ + "private/var/mobile/Library/Safari/History.db", + "private/var/mobile/Containers/Data/Application/*/Library/Safari/History.db", +] + +class SafariHistory(IOSExtraction): + """This module extracts all Safari visits and tries to detect potential + network injection attacks.""" + + def __init__(self, file_path=None, base_folder=None, output_folder=None, + fast_mode=False, log=None, results=[]): + super().__init__(file_path=file_path, base_folder=base_folder, + output_folder=output_folder, fast_mode=fast_mode, + log=log, results=results) + + def serialize(self, record): + return { + "timestamp": record["isodate"], + "module": self.__class__.__name__, + "event": "safari_history", + "data": f"Safari visit to {record['url']} (ID: {record['id']}, Visit ID: {record['visit_id']})", + } + + def _find_injections(self): + for result in self.results: + # We presume injections only happen on HTTP visits. + if not result["url"].lower().startswith("http://"): + continue + + # If there is no destination, no redirect happened. + if not result["redirect_destination"]: + continue + + origin_domain = URL(result["url"]).domain + + # We loop again through visits in order to find redirect record. + for redirect in self.results: + if redirect["visit_id"] != result["redirect_destination"]: + continue + + redirect_domain = URL(redirect["url"]).domain + # If the redirect destination is the same domain as the origin, + # it's most likely an HTTPS upgrade. + if origin_domain == redirect_domain: + continue + + self.log.info("Found HTTP redirect to different domain: \"%s\" -> \"%s\"", + origin_domain, redirect_domain) + + redirect_time = convert_mactime_to_unix(redirect["timestamp"]) + origin_time = convert_mactime_to_unix(result["timestamp"]) + elapsed_time = redirect_time - origin_time + elapsed_ms = elapsed_time.microseconds / 1000 + + if elapsed_time.seconds == 0: + self.log.warning("Redirect took less than a second! (%d milliseconds)", elapsed_ms) + + def check_indicators(self): + self._find_injections() + + if not self.indicators: + return + + for result in self.results: + if self.indicators.check_domain(result["url"]): + self.detected.append(result) + + def run(self): + self._find_ios_database(backup_ids=SAFARI_HISTORY_BACKUP_IDS, root_paths=SAFARI_HISTORY_ROOT_PATHS) + self.log.info("Found Safari history database at path: %s", self.file_path) + + conn = sqlite3.connect(self.file_path) + cur = conn.cursor() + cur.execute(""" + SELECT + history_items.id, + history_items.url, + history_visits.id, + history_visits.visit_time, + history_visits.redirect_source, + history_visits.redirect_destination + FROM history_items + JOIN history_visits ON history_visits.history_item = history_items.id + ORDER BY history_visits.visit_time; + """) + + items = [] + for item in cur: + items.append(dict( + id=item[0], + url=item[1], + visit_id=item[2], + timestamp=item[3], + isodate=convert_timestamp_to_iso(convert_mactime_to_unix(item[3])), + redirect_source=item[4], + redirect_destination=item[5] + )) + + cur.close() + conn.close() + + self.log.info("Extracted a total of %d history items", len(items)) + self.results = items diff --git a/mvt/ios/modules/fs/sms.py b/mvt/ios/modules/fs/sms.py new file mode 100644 index 0000000..1208074 --- /dev/null +++ b/mvt/ios/modules/fs/sms.py @@ -0,0 +1,97 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021 MVT Project Developers. +# See the file 'LICENSE' for usage and copying permissions, or find a copy at +# https://github.com/mvt-project/mvt/blob/main/LICENSE + +import sqlite3 +from base64 import b64encode + +from mvt.common.utils import check_for_links +from mvt.common.utils import convert_mactime_to_unix, convert_timestamp_to_iso + +from .base import IOSExtraction + +SMS_BACKUP_IDS = [ + "3d0d7e5fb2ce288813306e4d4636395e047a3d28", +] +SMS_ROOT_PATHS = [ + "private/var/mobile/Library/SMS/sms.db", +] + +class SMS(IOSExtraction): + """This module extracts all SMS messages containing links.""" + + def __init__(self, file_path=None, base_folder=None, output_folder=None, + fast_mode=False, log=None, results=[]): + super().__init__(file_path=file_path, base_folder=base_folder, + output_folder=output_folder, fast_mode=fast_mode, + log=log, results=results) + + def serialize(self, record): + text = record["text"].replace("\n", "\\n") + return { + "timestamp": record["isodate"], + "module": self.__class__.__name__, + "event": "sms_received", + "data": f"{record['service']}: {record['guid']} \"{text}\" from {record['phone_number']} ({record['account']})" + } + + def check_indicators(self): + if not self.indicators: + return + + for message in self.results: + if not "text" in message: + continue + + message_links = check_for_links(message["text"]) + if self.indicators.check_domains(message_links): + self.detected.append(message) + + def run(self): + self._find_ios_database(backup_ids=SMS_BACKUP_IDS, root_paths=SMS_ROOT_PATHS) + self.log.info("Found SMS database at path: %s", self.file_path) + + conn = sqlite3.connect(self.file_path) + cur = conn.cursor() + cur.execute(""" + SELECT + message.*, + handle.id as "phone_number" + FROM message, handle + WHERE handle.rowid = message.handle_id; + """) + names = [description[0] for description in cur.description] + + for item in cur: + message = dict() + for index, value in enumerate(item): + # We base64 escape some of the attributes that could contain + # binary data. + if (names[index] == "attributedBody" or + names[index] == "payload_data" or + names[index] == "message_summary_info") and value: + value = b64encode(value).decode() + + # We store the value of each column under the proper key. + message[names[index]] = value + + # We convert Mac's ridiculous timestamp format. + message["isodate"] = convert_timestamp_to_iso(convert_mactime_to_unix(message["date"])) + message["direction"] = ("sent" if message["is_from_me"] == 1 else "received") + + # Sometimes "text" is None instead of empty string. + if message["text"] is None: + message["text"] = "" + + # Extract links from the SMS message. + message_links = check_for_links(message["text"]) + + # If we find links in the messages or if they are empty we add them to the list. + if message_links or message["text"].strip() == "": + self.results.append(message) + + cur.close() + conn.close() + + self.log.info("Extracted a total of %d SMS messages containing links", len(self.results)) diff --git a/mvt/ios/modules/fs/sms_attachments.py b/mvt/ios/modules/fs/sms_attachments.py new file mode 100644 index 0000000..57cb8c1 --- /dev/null +++ b/mvt/ios/modules/fs/sms_attachments.py @@ -0,0 +1,84 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021 MVT Project Developers. +# See the file 'LICENSE' for usage and copying permissions, or find a copy at +# https://github.com/mvt-project/mvt/blob/main/LICENSE + +import sqlite3 +from base64 import b64encode + +from mvt.common.utils import check_for_links +from mvt.common.utils import convert_mactime_to_unix, convert_timestamp_to_iso + +from .base import IOSExtraction + +SMS_BACKUP_IDS = [ + "3d0d7e5fb2ce288813306e4d4636395e047a3d28", +] +SMS_ROOT_PATHS = [ + "private/var/mobile/Library/SMS/sms.db", +] + +class SMSAttachments(IOSExtraction): + """This module extracts all info about SMS/iMessage attachments.""" + + def __init__(self, file_path=None, base_folder=None, output_folder=None, + fast_mode=False, log=None, results=[]): + super().__init__(file_path=file_path, base_folder=base_folder, + output_folder=output_folder, fast_mode=fast_mode, + log=log, results=results) + + def serialize(self, record): + return { + "timestamp": record["isodate"], + "module": self.__class__.__name__, + "event": "sms_attachment", + "data": f"{record['service']}: Attachment '{record['transfer_name']}' {record['direction']} from {record['phone_number']} " + f"with {record['total_bytes']} bytes (is_sticker: {record['is_sticker']}, has_user_info: {record['has_user_info']})" + } + + def run(self): + self._find_ios_database(backup_ids=SMS_BACKUP_IDS, root_paths=SMS_ROOT_PATHS) + self.log.info("Found SMS database at path: %s", self.file_path) + + conn = sqlite3.connect(self.file_path) + cur = conn.cursor() + cur.execute(""" + SELECT + attachment.ROWID as "attachment_id", + attachment.*, + message.service as "service", + handle.id as "phone_number" + FROM attachment + LEFT JOIN message_attachment_join ON message_attachment_join.attachment_id = attachment.ROWID + LEFT JOIN message ON message.ROWID = message_attachment_join.message_id + LEFT JOIN handle ON handle.ROWID = message.handle_id + """) + names = [description[0] for description in cur.description] + + for item in cur: + attachment = dict() + for index, value in enumerate(item): + if (names[index] in ["user_info", "sticker_user_info", "attribution_info", + "ck_server_change_token_blob", "sr_ck_server_change_token_blob"]) and value: + value = b64encode(value).decode() + attachment[names[index]] = value + + # We convert Mac's ridiculous timestamp format. + attachment["isodate"] = convert_timestamp_to_iso(convert_mactime_to_unix(attachment["created_date"])) + attachment["start_date"] = convert_timestamp_to_iso(convert_mactime_to_unix(attachment["start_date"])) + attachment["direction"] = ("sent" if attachment["is_outgoing"] == 1 else "received") + attachment["has_user_info"] = attachment["user_info"] is not None + attachment["service"] = attachment["service"] or "Unknown" + attachment["filename"] = attachment["filename"] or "NULL" + + if (attachment["filename"].startswith("/var/tmp/") and attachment["filename"].endswith("-1") and + attachment["direction"] == "received"): + self.log.warn(f"Suspicious iMessage attachment '{attachment['filename']}' on {attachment['isodate']}") + self.detected.append(attachment) + + self.results.append(attachment) + + cur.close() + conn.close() + + self.log.info("Extracted a total of %d SMS attachments", len(self.results)) diff --git a/mvt/ios/modules/fs/version_history.py b/mvt/ios/modules/fs/version_history.py new file mode 100644 index 0000000..70dc054 --- /dev/null +++ b/mvt/ios/modules/fs/version_history.py @@ -0,0 +1,47 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021 MVT Project Developers. +# See the file 'LICENSE' for usage and copying permissions, or find a copy at +# https://github.com/mvt-project/mvt/blob/main/LICENSE + +import json +import datetime + +from mvt.common.utils import convert_timestamp_to_iso + +from .base import IOSExtraction + +IOS_ANALYTICS_JOURNAL_PATHS = [ + "private/var/db/analyticsd/Analytics-Journal-*.ips", +] + +class IOSVersionHistory(IOSExtraction): + """This module extracts iOS update history from Analytics Journal log files.""" + + def __init__(self, file_path=None, base_folder=None, output_folder=None, + fast_mode=False, log=None, results=[]): + super().__init__(file_path=file_path, base_folder=base_folder, + output_folder=output_folder, fast_mode=fast_mode, + log=log, results=results) + + def serialize(self, record): + return { + "timestamp": record["isodate"], + "module": self.__class__.__name__, + "event": "ios_version", + "data": f"Recorded iOS version {record['os_version']}", + } + + def run(self): + for found_path in self._find_paths(IOS_ANALYTICS_JOURNAL_PATHS): + with open(found_path, "r") as analytics_log: + log_line = json.loads(analytics_log.readline().strip()) + + timestamp = datetime.datetime.strptime(log_line["timestamp"], + "%Y-%m-%d %H:%M:%S.%f %z") + timestamp_utc = timestamp.astimezone(datetime.timezone.utc) + self.results.append({ + "isodate": convert_timestamp_to_iso(timestamp_utc), + "os_version": log_line["os_version"], + }) + + self.results = sorted(self.results, key=lambda entry: entry["isodate"]) diff --git a/mvt/ios/modules/fs/webkit_base.py b/mvt/ios/modules/fs/webkit_base.py new file mode 100644 index 0000000..2adb717 --- /dev/null +++ b/mvt/ios/modules/fs/webkit_base.py @@ -0,0 +1,40 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021 MVT Project Developers. +# See the file 'LICENSE' for usage and copying permissions, or find a copy at +# https://github.com/mvt-project/mvt/blob/main/LICENSE + +import os +import datetime + +from .base import IOSExtraction + +from mvt.common.utils import convert_timestamp_to_iso + +class WebkitBase(IOSExtraction): + """This class is a base for other WebKit-related modules.""" + + def check_indicators(self): + if not self.indicators: + return + + for item in self.results: + if self.indicators.check_domain(item["url"]): + self.detected.append(item) + + def _database_from_path(self, root_paths): + for found_path in self._find_paths(root_paths): + key = os.path.relpath(found_path, self.base_folder) + + for name in os.listdir(found_path): + if not name.startswith("http"): + continue + + name = name.replace("http_", "http://") + name = name.replace("https_", "https://") + url = name.split("_")[0] + + self.results.append(dict( + folder=key, + url=url, + isodate=convert_timestamp_to_iso(datetime.datetime.utcfromtimestamp(os.stat(found_path).st_mtime)), + )) diff --git a/mvt/ios/modules/fs/webkit_indexeddb.py b/mvt/ios/modules/fs/webkit_indexeddb.py new file mode 100644 index 0000000..55d65ec --- /dev/null +++ b/mvt/ios/modules/fs/webkit_indexeddb.py @@ -0,0 +1,35 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021 MVT Project Developers. +# See the file 'LICENSE' for usage and copying permissions, or find a copy at +# https://github.com/mvt-project/mvt/blob/main/LICENSE + +from .webkit_base import WebkitBase + +WEBKIT_INDEXEDDB_ROOT_PATHS = [ + "private/var/mobile/Containers/Data/Application/*/Library/WebKit/WebsiteData/IndexedDB", +] + +class WebkitIndexedDB(WebkitBase): + """This module looks extracts records from WebKit IndexedDB folders, + and checks them against any provided list of suspicious domains.""" + + slug = "webkit_indexeddb" + + def __init__(self, file_path=None, base_folder=None, output_folder=None, + fast_mode=False, log=None, results=[]): + super().__init__(file_path=file_path, base_folder=base_folder, + output_folder=output_folder, fast_mode=fast_mode, + log=log, results=results) + + def serialize(self, record): + return { + "timestamp": record["isodate"], + "module": self.__class__.__name__, + "event": "webkit_indexeddb", + "data": f"IndexedDB folder {record['folder']} containing file for URL {record['url']}", + } + + def run(self): + self._database_from_path(WEBKIT_INDEXEDDB_ROOT_PATHS) + self.log.info("Extracted a total of %d WebKit IndexedDB records", + len(self.results)) diff --git a/mvt/ios/modules/fs/webkit_localstorage.py b/mvt/ios/modules/fs/webkit_localstorage.py new file mode 100644 index 0000000..8b4fddb --- /dev/null +++ b/mvt/ios/modules/fs/webkit_localstorage.py @@ -0,0 +1,33 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021 MVT Project Developers. +# See the file 'LICENSE' for usage and copying permissions, or find a copy at +# https://github.com/mvt-project/mvt/blob/main/LICENSE + +from .webkit_base import WebkitBase + +WEBKIT_LOCALSTORAGE_ROOT_PATHS = [ + "private/var/mobile/Containers/Data/Application/*/Library/WebKit/WebsiteData/LocalStorage/", +] + +class WebkitLocalStorage(WebkitBase): + """This module looks extracts records from WebKit LocalStorage folders, + and checks them against any provided list of suspicious domains.""" + + def __init__(self, file_path=None, base_folder=None, output_folder=None, + fast_mode=False, log=None, results=[]): + super().__init__(file_path=file_path, base_folder=base_folder, + output_folder=output_folder, fast_mode=fast_mode, + log=log, results=results) + + def serialize(self, record): + return { + "timestamp": record["isodate"], + "module": self.__class__.__name__, + "event": "webkit_local_storage", + "data": f"WebKit Local Storage folder {record['folder']} containing file for URL {record['url']}", + } + + def run(self): + self._database_from_path(WEBKIT_LOCALSTORAGE_ROOT_PATHS) + self.log.info("Extracted a total of %d records from WebKit Local Storages", + len(self.results)) diff --git a/mvt/ios/modules/fs/webkit_safariviewservice.py b/mvt/ios/modules/fs/webkit_safariviewservice.py new file mode 100644 index 0000000..33f993c --- /dev/null +++ b/mvt/ios/modules/fs/webkit_safariviewservice.py @@ -0,0 +1,23 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021 MVT Project Developers. +# See the file 'LICENSE' for usage and copying permissions, or find a copy at +# https://github.com/mvt-project/mvt/blob/main/LICENSE + +from .webkit_base import WebkitBase + +WEBKIT_SAFARIVIEWSERVICE_ROOT_PATHS = [ + "private/var/mobile/Containers/Data/Application/*/SystemData/com.apple.SafariViewService/Library/WebKit/WebsiteData/", +] + +class WebkitSafariViewService(WebkitBase): + """This module looks extracts records from WebKit LocalStorage folders, + and checks them against any provided list of suspicious domains.""" + + def __init__(self, file_path=None, base_folder=None, output_folder=None, + fast_mode=False, log=None, results=[]): + super().__init__(file_path=file_path, base_folder=base_folder, + output_folder=output_folder, fast_mode=fast_mode, + log=log, results=results) + + def run(self): + self._database_from_path(WEBKIT_SAFARIVIEWSERVICE_ROOT_PATHS) diff --git a/mvt/ios/modules/fs/webkit_session_resource_log.py b/mvt/ios/modules/fs/webkit_session_resource_log.py new file mode 100644 index 0000000..299306d --- /dev/null +++ b/mvt/ios/modules/fs/webkit_session_resource_log.py @@ -0,0 +1,136 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021 MVT Project Developers. +# See the file 'LICENSE' for usage and copying permissions, or find a copy at +# https://github.com/mvt-project/mvt/blob/main/LICENSE + +import os +import glob +import biplist + +from mvt.common.utils import convert_timestamp_to_iso + +from .base import IOSExtraction + +WEBKIT_SESSION_RESOURCE_LOG_BACKUP_IDS = [ + "a500ee38053454a02e990957be8a251935e28d3f", +] + +WEBKIT_SESSION_RESOURCE_LOG_ROOT_PATHS = [ + "private/var/mobile/Containers/Data/Application/*/SystemData/com.apple.SafariViewService/Library/WebKit/WebsiteData/full_browsing_session_resourceLog.plist", + "private/var/mobile/Containers/Data/Application/*/Library/WebKit/WebsiteData/ResourceLoadStatistics/full_browsing_session_resourceLog.plist", + "private/var/mobile/Library/WebClips/*/Storage/full_browsing_session_resourceLog.plist", +] + +class WebkitSessionResourceLog(IOSExtraction): + """This module extracts records from WebKit browsing session + resource logs, and checks them against any provided list of + suspicious domains.""" + + def __init__(self, file_path=None, base_folder=None, output_folder=None, + fast_mode=False, log=None, results=[]): + super().__init__(file_path=file_path, base_folder=base_folder, + output_folder=output_folder, fast_mode=fast_mode, + log=log, results=results) + + def _extract_browsing_stats(self, file_path): + items = [] + + file_plist = biplist.readPlist(file_path) + if "browsingStatistics" not in file_plist: + return items + + browsing_stats = file_plist["browsingStatistics"] + + for item in browsing_stats: + items.append(dict( + origin=item.get("PrevalentResourceOrigin", ""), + redirect_source=item.get("topFrameUniqueRedirectsFrom", ""), + redirect_destination=item.get("topFrameUniqueRedirectsTo", ""), + subframe_under_origin=item.get("subframeUnderTopFrameOrigins", ""), + subresource_under_origin=item.get("subresourceUnderTopFrameOrigins", ""), + user_interaction=item.get("hadUserInteraction"), + most_recent_interaction=convert_timestamp_to_iso(item["mostRecentUserInteraction"]), + last_seen=convert_timestamp_to_iso(item["lastSeen"]), + )) + + return items + + @staticmethod + def _extract_domains(entries): + if not entries: + return [] + + domains = [] + for entry in entries: + if "origin" in entry: + domains.append(entry["origin"]) + if "domain" in entry: + domains.append(entry["domain"]) + + return domains + + def check_indicators(self): + for key, entries in self.results.items(): + for entry in entries: + source_domains = self._extract_domains(entry["redirect_source"]) + destination_domains = self._extract_domains(entry["redirect_destination"]) + + # TODO: Currently not used. + # subframe_origins = self._extract_domains(entry["subframe_under_origin"]) + # subresource_domains = self._extract_domains(entry["subresource_under_origin"]) + + all_origins = set([entry["origin"]] + source_domains + destination_domains) + + if self.indicators.check_domains(all_origins): + self.detected.append(entry) + + redirect_path = "" + if len(source_domains) > 0: + redirect_path += "SOURCE: " + for idx, item in enumerate(source_domains): + source_domains[idx] = f"\"{item}\"" + + redirect_path += ", ".join(source_domains) + redirect_path += " -> " + + redirect_path += f"ORIGIN: \"{entry['origin']}\"" + + if len(destination_domains) > 0: + redirect_path += " -> " + redirect_path += "DESTINATION: " + for idx, item in enumerate(destination_domains): + destination_domains[idx] = f"\"{item}\"" + + redirect_path += ", ".join(destination_domains) + + self.log.warning("Found HTTP redirect between suspicious domains: %s", redirect_path) + + def _find_paths(self, root_paths): + results = {} + for root_path in root_paths: + for found_path in glob.glob(os.path.join(self.base_folder, root_path)): + if not os.path.exists(found_path): + continue + + key = os.path.relpath(found_path, self.base_folder) + if key not in results: + results[key] = [] + + return results + + def run(self): + self.results = {} + + try: + self._find_ios_database(backup_ids=WEBKIT_SESSION_RESOURCE_LOG_BACKUP_IDS) + except FileNotFoundError: + pass + else: + if self.file_path: + self.results[self.file_path] = self._extract_browsing_stats(self.file_path) + return + + self.results = self._find_paths(root_paths=WEBKIT_SESSION_RESOURCE_LOG_ROOT_PATHS) + for log_file in self.results.keys(): + self.log.info("Found Safari browsing session resource log at path: %s", log_file) + self.results[log_file] = self._extract_browsing_stats(os.path.join(self.base_folder, log_file)) diff --git a/mvt/ios/modules/fs/whatsapp.py b/mvt/ios/modules/fs/whatsapp.py new file mode 100644 index 0000000..34db73a --- /dev/null +++ b/mvt/ios/modules/fs/whatsapp.py @@ -0,0 +1,83 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021 MVT Project Developers. +# See the file 'LICENSE' for usage and copying permissions, or find a copy at +# https://github.com/mvt-project/mvt/blob/main/LICENSE + +import sqlite3 +import logging + +from mvt.common.utils import convert_mactime_to_unix, convert_timestamp_to_iso, check_for_links + +from .base import IOSExtraction + +log = logging.getLogger(__name__) + +WHATSAPP_BACKUP_IDS = [ + "7c7fba66680ef796b916b067077cc246adacf01d", +] +WHATSAPP_ROOT_PATHS = [ + "private/var/mobile/Containers/Shared/AppGroup/*/ChatStorage.sqlite", +] + +class Whatsapp(IOSExtraction): + """This module extracts all WhatsApp messages containing links.""" + + def __init__(self, file_path=None, base_folder=None, output_folder=None, + fast_mode=False, log=None, results=[]): + super().__init__(file_path=file_path, base_folder=base_folder, + output_folder=output_folder, fast_mode=fast_mode, + log=log, results=results) + + def serialize(self, record): + text = record["ZTEXT"].replace("\n", "\\n") + return { + "timestamp": record["isodate"], + "module": self.__class__.__name__, + "event": "message", + "data": f"{text} from {record['ZFROMJID']}" + } + + def check_indicators(self): + if not self.indicators: + return + + for message in self.results: + if not "ZTEXT" in message: + continue + + message_links = check_for_links(message["ZTEXT"]) + if self.indicators.check_domains(message_links): + self.detected.append(message) + + def run(self): + self._find_ios_database(backup_ids=WHATSAPP_BACKUP_IDS, root_paths=WHATSAPP_ROOT_PATHS) + + log.info("Found WhatsApp database at path: %s", self.file_path) + + conn = sqlite3.connect(self.file_path) + cur = conn.cursor() + cur.execute("SELECT * FROM ZWAMESSAGE;") + names = [description[0] for description in cur.description] + + for message in cur: + new_message = dict() + for index, value in enumerate(message): + new_message[names[index]] = value + + if not new_message["ZTEXT"]: + continue + + # We convert Mac's silly timestamp again. + new_message["isodate"] = convert_timestamp_to_iso(convert_mactime_to_unix(new_message["ZMESSAGEDATE"])) + + # Extract links from the WhatsApp message. + message_links = check_for_links(new_message["ZTEXT"]) + + # If we find mesages, or if there's an empty message we add it to the list. + if new_message["ZTEXT"] and (message_links or new_message["ZTEXT"].strip() == ""): + self.results.append(new_message) + + cur.close() + conn.close() + + log.info("Extracted a total of %d WhatsApp messages containing links", len(self.results)) diff --git a/mvt/ios/versions.py b/mvt/ios/versions.py new file mode 100644 index 0000000..6ca0b50 --- /dev/null +++ b/mvt/ios/versions.py @@ -0,0 +1,192 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021 MVT Project Developers. +# See the file 'LICENSE' for usage and copying permissions, or find a copy at +# https://github.com/mvt-project/mvt/blob/main/LICENSE + +IPHONE_IOS_VERSIONS = [ + {"build": "7E18", "version": "3.1.3"}, + {"build": "7D11", "version": "3.1.2"}, + {"build": "7C144", "version": "3.1"}, + {"build": "7A400", "version": "3.0.1"}, + {"build": "7A341", "version": "3.0"}, + {"build": "5H11", "version": "2.2.1"}, + {"build": "5G77", "version": "2.2"}, + {"build": "5F136", "version": "2.1"}, + {"build": "5C1", "version": "2.0.2"}, + {"build": "5B108", "version": "2.0.1"}, + {"build": "5A347", "version": "2.0"}, + {"build": "4A102", "version": "1.1.4"}, + {"build": "4A93", "version": "1.1.3"}, + {"build": "3B48b", "version": "1.1.2"}, + {"build": "3A109a", "version": "1.1.1"}, + {"build": "1C28", "version": "1.0.2"}, + {"build": "1C25", "version": "1.0.1"}, + {"build": "1A543a", "version": "1.0"}, + {"build": "8C148", "version": "4.2"}, + {"build": "8B117", "version": "4.1"}, + {"build": "8A306", "version": "4.0.1"}, + {"build": "8A293", "version": "4.0"}, + {"build": "10B500", "version": "6.1.6"}, + {"build": "10B329", "version": "6.1.3"}, + {"build": "10B146", "version": "6.1.2"}, + {"build": "10B141", "version": "6.1"}, + {"build": "10A523", "version": "6.0.1"}, + {"build": "10A403", "version": "6.0"}, + {"build": "9B206", "version": "5.1.1"}, + {"build": "9B176", "version": "5.1"}, + {"build": "9A405", "version": "5.0.1"}, + {"build": "9A334", "version": "5.0"}, + {"build": "8L1", "version": "4.3.5"}, + {"build": "8K2", "version": "4.3.4"}, + {"build": "8J2", "version": "4.3.3"}, + {"build": "8F190", "version": "4.3"}, + {"build": "8C148a", "version": "4.2.1"}, + {"build": "11D257", "version": "7.1.2"}, + {"build": "11D201", "version": "7.1.1"}, + {"build": "11D169", "version": "7.1"}, + {"build": "11B651", "version": "7.0.6"}, + {"build": "11B554a", "version": "7.0.4"}, + {"build": "11B511", "version": "7.0.3"}, + {"build": "10B144", "version": "6.1"}, + {"build": "9B208", "version": "5.1.1"}, + {"build": "8C148", "version": "4.2.1"}, + {"build": "11D167", "version": "7.1"}, + {"build": "8E600", "version": "4.2.10"}, + {"build": "8E501", "version": "4.2.9"}, + {"build": "8E401", "version": "4.2.8"}, + {"build": "13G37", "version": "9.3.6"}, + {"build": "13G36", "version": "9.3.5"}, + {"build": "13G35", "version": "9.3.4"}, + {"build": "13G34", "version": "9.3.3"}, + {"build": "13F69", "version": "9.3.2"}, + {"build": "13E238", "version": "9.3.1"}, + {"build": "13E237", "version": "9.3"}, + {"build": "13E233", "version": "9.3"}, + {"build": "13D15", "version": "9.2.1"}, + {"build": "13C75", "version": "9.2"}, + {"build": "13B143", "version": "9.1"}, + {"build": "13A452", "version": "9.0.2"}, + {"build": "13A404", "version": "9.0.1"}, + {"build": "13A344", "version": "9.0"}, + {"build": "12H321", "version": "8.4.1"}, + {"build": "12H143", "version": "8.4"}, + {"build": "12F70", "version": "8.3"}, + {"build": "12D508", "version": "8.2"}, + {"build": "12B466", "version": "8.1.3"}, + {"build": "12B440", "version": "8.1.2"}, + {"build": "12B435", "version": "8.1.1"}, + {"build": "12B411", "version": "8.1"}, + {"build": "12A405", "version": "8.0.2"}, + {"build": "12A402", "version": "8.0.1"}, + {"build": "12A365", "version": "8.0"}, + {"build": "10B145", "version": "6.1.1"}, + {"build": "10B142", "version": "6.1"}, + {"build": "9B179", "version": "5.1"}, + {"build": "9A406", "version": "5.0.1"}, + {"build": "14G61", "version": "10.3.4"}, + {"build": "14G60", "version": "10.3.3"}, + {"build": "14F89", "version": "10.3.2"}, + {"build": "14E304", "version": "10.3.1"}, + {"build": "14E277", "version": "10.3"}, + {"build": "14D27", "version": "10.2.1"}, + {"build": "14C92", "version": "10.2"}, + {"build": "14B150", "version": "10.1.1"}, + {"build": "14B100", "version": "10.1.1"}, + {"build": "14B72", "version": "10.1"}, + {"build": "14A456", "version": "10.0.2"}, + {"build": "14A403", "version": "10.0.1"}, + {"build": "10B350", "version": "6.1.4"}, + {"build": "10B143", "version": "6.1"}, + {"build": "10A551", "version": "6.0.2"}, + {"build": "10A525", "version": "6.0.1"}, + {"build": "10A405", "version": "6.0"}, + {"build": "11B601", "version": "7.0.5"}, + {"build": "18F72", "version": "14.6"}, + {"build": "18E199", "version": "14.5"}, + {"build": "18E212", "version": "14.5.1"}, + {"build": "18D52", "version": "14.4"}, + {"build": "18D61", "version": "14.4.1"}, + {"build": "18D70", "version": "14.4.2"}, + {"build": "18C66", "version": "14.3"}, + {"build": "18B92", "version": "14.2"}, + {"build": "18A8395", "version": "14.1"}, + {"build": "18A393", "version": "14.0.1"}, + {"build": "18A373", "version": "14.0"}, + {"build": "17H35", "version": "13.7"}, + {"build": "17G80", "version": "13.6.1"}, + {"build": "17G68", "version": "13.6"}, + {"build": "17F80", "version": "13.5.1"}, + {"build": "17F75", "version": "13.5"}, + {"build": "17E262", "version": "13.4.1"}, + {"build": "17E255", "version": "13.4"}, + {"build": "17C54", "version": "13.3"}, + {"build": "17D50", "version": "13.3.1"}, + {"build": "17B111", "version": "13.2.3"}, + {"build": "17B102", "version": "13.2.2"}, + {"build": "17B84", "version": "13.2"}, + {"build": "17A878", "version": "13.1.3"}, + {"build": "17A860", "version": "13.1.2"}, + {"build": "17A854", "version": "13.1.1"}, + {"build": "17A844", "version": "13.1"}, + {"build": "17A577", "version": "13.0"}, + {"build": "16H22", "version": "12.5.1"}, + {"build": "16H20", "version": "12.5"}, + {"build": "16H5", "version": "12.4.9"}, + {"build": "16G201", "version": "12.4.8"}, + {"build": "16G192", "version": "12.4.7"}, + {"build": "16G183", "version": "12.4.6"}, + {"build": "16G161", "version": "12.4.5"}, + {"build": "16G130", "version": "12.4.3"}, + {"build": "16G114", "version": "12.4.2"}, + {"build": "16G102", "version": "12.4.1"}, + {"build": "16G77", "version": "12.4"}, + {"build": "16F203", "version": "12.3.1"}, + {"build": "16F156", "version": "12.3"}, + {"build": "16E227", "version": "12.2"}, + {"build": "16D57", "version": "12.1.4"}, + {"build": "16D39", "version": "12.1.3"}, + {"build": "16C104", "version": "12.1.2"}, + {"build": "16C101", "version": "12.1.2"}, + {"build": "16C50", "version": "12.1.1"}, + {"build": "16B92", "version": "12.1"}, + {"build": "16A404", "version": "12.0.1"}, + {"build": "16A366", "version": "12.0"}, + {"build": "15G77", "version": "11.4.1"}, + {"build": "15F79", "version": "11.4"}, + {"build": "15E302", "version": "11.3.1"}, + {"build": "15E216", "version": "11.3"}, + {"build": "15D100", "version": "11.2.6"}, + {"build": "15D60", "version": "11.2.5"}, + {"build": "15C202", "version": "11.2.2"}, + {"build": "15C153", "version": "11.2.1"}, + {"build": "15C114", "version": "11.2"}, + {"build": "15B202", "version": "11.1.2"}, + {"build": "15B150", "version": "11.1.1"}, + {"build": "15B93", "version": "11.1"}, + {"build": "15A432", "version": "11.0.3"}, + {"build": "15A421", "version": "11.0.2"}, + {"build": "15A402", "version": "11.0.1"}, + {"build": "15A372", "version": "11.0"}, + {"build": "13D20", "version": "9.2.1"}, + {"build": "12B436", "version": "8.1.1"}, + {"build": "12A366", "version": "8.0"}, + {"build": "13E234", "version": "9.3"}, + {"build": "13A405", "version": "9.0.1"}, + {"build": "13A342", "version": "9.0"}, + {"build": "13A343", "version": "9.0"}, + {"build": "14B72c", "version": "10.1"}, + {"build": "14A551", "version": "10.0.3"}, + {"build": "16F250", "version": "12.3.2"}, + {"build": "17A861", "version": "13.1.2"}, + {"build": "16D40", "version": "12.1.3"}, + {"build": "16A405", "version": "12.0.1"}, + {"build": "16B94", "version": "12.1"}, + {"build": "16B93", "version": "12.1"}, + {"build": "17E8258", "version": "13.4.1"} +] + +def find_version_by_build(build): + build = build.upper() + for version in IPHONE_IOS_VERSIONS: + if build == version["build"]: + return version["version"] diff --git a/setup.py b/setup.py new file mode 100755 index 0000000..d306607 --- /dev/null +++ b/setup.py @@ -0,0 +1,65 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021 MVT Project Developers. +# See the file 'LICENSE' for usage and copying permissions, or find a copy at +# https://github.com/mvt-project/mvt/blob/main/LICENSE + +import os +from setuptools import setup, find_packages + +__package_name__ = "mvt" +__version__ = "1.0" +__description__ = "Mobile Verification Toolkit" + +this_directory = os.path.abspath(os.path.dirname(__file__)) +readme_path = os.path.join(this_directory, "README.md") +with open(readme_path, encoding="utf-8") as handle: + long_description = handle.read() + +requires = ( + # Base dependencies: + "click", + "rich", + "tld", + "tqdm", + "requests", + "simplejson", + # iOS dependencies: + "biplist", + "iOSbackup", + # Android dependencies: + "adb-shell", + "libusb1", +) + +def get_package_data(package): + walk = [(dirpath.replace(package + os.sep, "", 1), filenames) + for dirpath, dirnames, filenames in os.walk(package) + if not os.path.exists(os.path.join(dirpath, "__init__.py"))] + + filepaths = [] + for base, filenames in walk: + filepaths.extend([os.path.join(base, filename) + for filename in filenames]) + return {package: filepaths} + +setup( + name=__package_name__, + version=__version__, + description=__description__, + long_description=long_description, + + entry_points={ + "console_scripts": [ + "mvt-ios = mvt.ios:cli", + "mvt-android = mvt.android:cli", + ], + }, + install_requires=requires, + packages=find_packages(), + package_data=get_package_data("mvt"), + include_package_data=True, + keywords="security mobile forensics malware", + license="MVT", + classifiers=[ + ], +)