mirror of
https://github.com/mvt-project/mvt.git
synced 2024-06-24 05:18:55 +00:00
Changed mvt-android download-apks to only fetch non-system packages
This commit is contained in:
Nex
parent
67d8820cc9
commit
79f313827f
|
|
@ -39,7 +39,7 @@ def cli():
|
|||
@cli.command("download-apks", help="Download all or non-safelisted installed APKs installed on the device")
|
||||
@click.option("--serial", "-s", type=str, help=HELP_MSG_SERIAL)
|
||||
@click.option("--all-apks", "-a", is_flag=True,
|
||||
help="Extract all packages installed on the phone, even those marked as safe")
|
||||
help="Extract all packages installed on the phone, including system packages")
|
||||
@click.option("--virustotal", "-v", is_flag=True, help="Check packages on VirusTotal")
|
||||
@click.option("--koodous", "-k", is_flag=True, help="Check packages on Koodous")
|
||||
@click.option("--all-checks", "-A", is_flag=True, help="Run all available checks")
|
||||
|
|
@ -65,7 +65,8 @@ def download_apks(ctx, all_apks, virustotal, koodous, all_checks, output, from_f
|
|||
log.critical("Unable to create output folder %s: %s", output, e)
|
||||
ctx.exit(1)
|
||||
|
||||
download = DownloadAPKs(output_folder=output, all_apks=all_apks)
|
||||
download = DownloadAPKs(output_folder=output, all_apks=all_apks,
|
||||
log=logging.getLogger(DownloadAPKs.__module__))
|
||||
if serial:
|
||||
download.serial = serial
|
||||
download.run()
|
||||
|
|
|
|||
|
|
@ -1,182 +0,0 @@
|
|||
android
|
||||
android.auto_generated_rro__
|
||||
android.autoinstalls.config.google.nexus
|
||||
com.android.backupconfirm
|
||||
com.android.bips
|
||||
com.android.bluetooth
|
||||
com.android.bluetoothmidiservice
|
||||
com.android.bookmarkprovider
|
||||
com.android.calllogbackup
|
||||
com.android.captiveportallogin
|
||||
com.android.carrierconfig
|
||||
com.android.carrierdefaultapp
|
||||
com.android.cellbroadcastreceiver
|
||||
com.android.certinstaller
|
||||
com.android.chrome
|
||||
com.android.companiondevicemanager
|
||||
com.android.connectivity.metrics
|
||||
com.android.cts.ctsshim
|
||||
com.android.cts.priv.ctsshim
|
||||
com.android.defcontainer
|
||||
com.android.documentsui
|
||||
com.android.dreams.basic
|
||||
com.android.egg
|
||||
com.android.emergency
|
||||
com.android.externalstorage
|
||||
com.android.facelock
|
||||
com.android.hotwordenrollment
|
||||
com.android.hotwordenrollment.okgoogle
|
||||
com.android.hotwordenrollment.tgoogle
|
||||
com.android.hotwordenrollment.xgoogle
|
||||
com.android.htmlviewer
|
||||
com.android.inputdevices
|
||||
com.android.keychain
|
||||
com.android.location.fused
|
||||
com.android.managedprovisioning
|
||||
com.android.mms.service
|
||||
com.android.mtp
|
||||
com.android.musicfx
|
||||
com.android.nfc
|
||||
com.android.omadm.service
|
||||
com.android.pacprocessor
|
||||
com.android.phone
|
||||
com.android.printspooler
|
||||
com.android.providers.blockednumber
|
||||
com.android.providers.calendar
|
||||
com.android.providers.contacts
|
||||
com.android.providers.downloads
|
||||
com.android.providers.downloads.ui
|
||||
com.android.providers.media
|
||||
com.android.providers.partnerbookmarks
|
||||
com.android.providers.settings
|
||||
com.android.providers.telephony
|
||||
com.android.providers.userdictionary
|
||||
com.android.proxyhandler
|
||||
com.android.retaildemo
|
||||
com.android.safetyregulatoryinfo
|
||||
com.android.sdm.plugins.connmo
|
||||
com.android.sdm.plugins.dcmo
|
||||
com.android.sdm.plugins.diagmon
|
||||
com.android.sdm.plugins.sprintdm
|
||||
com.android.server.telecom
|
||||
com.android.service.ims
|
||||
com.android.service.ims.presence
|
||||
com.android.settings
|
||||
com.android.sharedstoragebackup
|
||||
com.android.shell
|
||||
com.android.statementservice
|
||||
com.android.stk
|
||||
com.android.systemui
|
||||
com.android.systemui.theme.dark
|
||||
com.android.vending
|
||||
com.android.vpndialogs
|
||||
com.android.vzwomatrigger
|
||||
com.android.wallpaperbackup
|
||||
com.android.wallpaper.livepicker
|
||||
com.breel.wallpapers
|
||||
com.customermobile.preload.vzw
|
||||
com.google.android.apps.cloudprint
|
||||
com.google.android.apps.docs
|
||||
com.google.android.apps.docs.editors.docs
|
||||
com.google.android.apps.enterprise.dmagent
|
||||
com.google.android.apps.gcs
|
||||
com.google.android.apps.helprtc
|
||||
com.google.android.apps.inputmethod.hindi
|
||||
com.google.android.apps.maps
|
||||
com.google.android.apps.messaging
|
||||
com.google.android.apps.nexuslauncher
|
||||
com.google.android.apps.photos
|
||||
com.google.android.apps.pixelmigrate
|
||||
com.google.android.apps.tachyon
|
||||
com.google.android.apps.turbo
|
||||
com.google.android.apps.tycho
|
||||
com.google.android.apps.wallpaper
|
||||
com.google.android.apps.wallpaper.nexus
|
||||
com.google.android.apps.work.oobconfig
|
||||
com.google.android.apps.youtube.vr
|
||||
com.google.android.asdiv
|
||||
com.google.android.backuptransport
|
||||
com.google.android.calculator
|
||||
com.google.android.calendar
|
||||
com.google.android.carrier
|
||||
com.google.android.carrier.authdialog
|
||||
com.google.android.carrierentitlement
|
||||
com.google.android.carriersetup
|
||||
com.google.android.configupdater
|
||||
com.google.android.contacts
|
||||
com.google.android.deskclock
|
||||
com.google.android.dialer
|
||||
com.google.android.euicc
|
||||
com.google.android.ext.services
|
||||
com.google.android.ext.shared
|
||||
com.google.android.feedback
|
||||
com.google.android.gm
|
||||
com.google.android.gms
|
||||
com.google.android.gms.policy_auth
|
||||
com.google.android.gms.policy_sidecar_o
|
||||
com.google.android.gms.setup
|
||||
com.google.android.GoogleCamera
|
||||
com.google.android.googlequicksearchbox
|
||||
com.google.android.gsf
|
||||
com.google.android.gsf.login
|
||||
com.google.android.hardwareinfo
|
||||
com.google.android.hiddenmenu
|
||||
com.google.android.ims
|
||||
com.google.android.inputmethod.japanese
|
||||
com.google.android.inputmethod.korean
|
||||
com.google.android.inputmethod.latin
|
||||
com.google.android.inputmethod.pinyin
|
||||
com.google.android.instantapps.supervisor
|
||||
com.google.android.keep
|
||||
com.google.android.marvin.talkback
|
||||
com.google.android.music
|
||||
com.google.android.nexusicons
|
||||
com.google.android.onetimeinitializer
|
||||
com.google.android.packageinstaller
|
||||
com.google.android.partnersetup
|
||||
com.google.android.printservice.recommendation
|
||||
com.google.android.setupwizard
|
||||
com.google.android.soundpicker
|
||||
com.google.android.storagemanager
|
||||
com.google.android.syncadapters.contacts
|
||||
com.google.android.tag
|
||||
com.google.android.talk
|
||||
com.google.android.tetheringentitlement
|
||||
com.google.android.theme.pixel
|
||||
com.google.android.tts
|
||||
com.google.android.videos
|
||||
com.google.android.vr.home
|
||||
com.google.android.vr.inputmethod
|
||||
com.google.android.webview
|
||||
com.google.android.wfcactivation
|
||||
com.google.android.youtube
|
||||
com.google.ar.core
|
||||
com.google.intelligence.sense
|
||||
com.google.modemservice
|
||||
com.google.pixel.wahoo.gfxdrv
|
||||
com.google.SSRestartDetector
|
||||
com.google.tango
|
||||
com.google.vr.apps.ornament
|
||||
com.google.vr.vrcore
|
||||
com.htc.omadm.trigger
|
||||
com.qti.qualcomm.datastatusnotification
|
||||
com.qualcomm.atfwd
|
||||
com.qualcomm.embms
|
||||
com.qualcomm.fastdormancy
|
||||
com.qualcomm.ltebc_vzw
|
||||
com.qualcomm.qcrilmsgtunnel
|
||||
com.qualcomm.qti.ims
|
||||
com.qualcomm.qti.networksetting
|
||||
com.qualcomm.qti.telephonyservice
|
||||
com.qualcomm.qti.uceShimService
|
||||
com.qualcomm.shutdownlistner
|
||||
com.qualcomm.timeservice
|
||||
com.qualcomm.vzw_api
|
||||
com.quicinc.cne.CNEService
|
||||
com.verizon.llkagent
|
||||
com.verizon.mips.services
|
||||
com.verizon.obdm
|
||||
com.verizon.obdm_permissions
|
||||
com.verizon.services
|
||||
com.vzw.apnlib
|
||||
qualcomm.com.vzw_msdc_api
|
||||
|
|
@ -14,6 +14,7 @@ from mvt.common.module import InsufficientPrivileges
|
|||
from mvt.common.utils import get_sha256_from_file_path
|
||||
|
||||
from .modules.adb.base import AndroidExtraction
|
||||
from .modules.adb.packages import Packages
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -29,33 +30,23 @@ class PullProgress(tqdm):
|
|||
self.update(current - self.n)
|
||||
|
||||
|
||||
class Package:
|
||||
"""Package indicates a package name and all the files associated with it."""
|
||||
|
||||
def __init__(self, name, files=None):
|
||||
self.name = name
|
||||
self.files = files or []
|
||||
|
||||
|
||||
class DownloadAPKs(AndroidExtraction):
|
||||
"""DownloadAPKs is the main class operating the download of APKs
|
||||
from the device."""
|
||||
|
||||
def __init__(self, output_folder=None, all_apks=False, packages=None):
|
||||
def __init__(self, output_folder=None, all_apks=False, log=None,
|
||||
packages=None):
|
||||
"""Initialize module.
|
||||
:param output_folder: Path to the folder where data should be stored
|
||||
:param all_apks: Boolean indicating whether to download all packages
|
||||
or filter known-goods
|
||||
:param packages: Provided list of packages, typically for JSON checks
|
||||
"""
|
||||
super().__init__(file_path=None, base_folder=None,
|
||||
output_folder=output_folder)
|
||||
super().__init__(output_folder=output_folder, log=log)
|
||||
|
||||
self.output_folder_apk = None
|
||||
self.packages = packages or []
|
||||
self.packages = packages
|
||||
self.all_apks = all_apks
|
||||
|
||||
self._safe_packages = []
|
||||
self.output_folder_apk = None
|
||||
|
||||
@classmethod
|
||||
def from_json(cls, json_path):
|
||||
|
|
@ -63,55 +54,9 @@ class DownloadAPKs(AndroidExtraction):
|
|||
:param json_path: Path to the apks.json file to parse.
|
||||
"""
|
||||
with open(json_path, "r") as handle:
|
||||
data = json.load(handle)
|
||||
|
||||
packages = []
|
||||
for entry in data:
|
||||
package = Package(entry["name"], entry["files"])
|
||||
packages.append(package)
|
||||
|
||||
packages = json.load(handle)
|
||||
return cls(packages=packages)
|
||||
|
||||
def _load_safe_packages(self):
|
||||
"""Load known-good package names.
|
||||
"""
|
||||
safe_packages_path = os.path.join("data", "safe_packages.txt")
|
||||
safe_packages_string = pkg_resources.resource_string(__name__, safe_packages_path)
|
||||
safe_packages_list = safe_packages_string.decode("utf-8").split("\n")
|
||||
self._safe_packages.extend(safe_packages_list)
|
||||
|
||||
def _clean_output(self, output):
|
||||
"""Clean adb shell command output.
|
||||
:param output: Command output to clean.
|
||||
"""
|
||||
return output.strip().replace("package:", "")
|
||||
|
||||
def get_packages(self):
|
||||
"""Retrieve package names from the device using adb.
|
||||
"""
|
||||
log.info("Retrieving package names ...")
|
||||
|
||||
if not self.all_apks:
|
||||
self._load_safe_packages()
|
||||
|
||||
output = self._adb_command("pm list packages")
|
||||
total = 0
|
||||
for line in output.split("\n"):
|
||||
package_name = self._clean_output(line)
|
||||
if package_name == "":
|
||||
continue
|
||||
|
||||
total += 1
|
||||
|
||||
if not self.all_apks and package_name in self._safe_packages:
|
||||
continue
|
||||
|
||||
if package_name not in self.packages:
|
||||
self.packages.append(Package(package_name))
|
||||
|
||||
log.info("There are %d packages installed on the device. I selected %d for inspection.",
|
||||
total, len(self.packages))
|
||||
|
||||
def pull_package_file(self, package_name, remote_path):
|
||||
"""Pull files related to specific package from the device.
|
||||
:param package_name: Name of the package to download
|
||||
|
|
@ -153,6 +98,18 @@ class DownloadAPKs(AndroidExtraction):
|
|||
|
||||
return local_path
|
||||
|
||||
def get_packages(self):
|
||||
"""Use the Packages adb module to retrieve the list of packages.
|
||||
We reuse the same extraction logic to then download the APKs.
|
||||
"""
|
||||
self.log.info("Retrieving list of installed packages...")
|
||||
|
||||
m = Packages()
|
||||
m.log = self.log
|
||||
m.run()
|
||||
|
||||
self.packages = m.results
|
||||
|
||||
def pull_packages(self):
|
||||
"""Download all files of all selected packages from the device.
|
||||
"""
|
||||
|
|
@ -161,26 +118,47 @@ class DownloadAPKs(AndroidExtraction):
|
|||
if not os.path.exists(self.output_folder):
|
||||
os.mkdir(self.output_folder)
|
||||
|
||||
# If the user provided the flag --all-apks we select all packages.
|
||||
packages_selection = []
|
||||
if self.all_apks:
|
||||
log.info("Selected all %d available packages", len(self.packages))
|
||||
packages_selection = self.packages
|
||||
else:
|
||||
# Otherwise we loop through the packages and get only those that
|
||||
# are not marked as system.
|
||||
for package in self.packages:
|
||||
if not package.get("system", False):
|
||||
packages_selection.append(package)
|
||||
|
||||
log.info("Selected only %d packages which are not marked as system",
|
||||
len(packages_selection))
|
||||
|
||||
if len(packages_selection) == 0:
|
||||
log.info("No packages were selected for download")
|
||||
return
|
||||
|
||||
log.info("Downloading packages from device. This might take some time ...")
|
||||
|
||||
self.output_folder_apk = os.path.join(self.output_folder, "apks")
|
||||
if not os.path.exists(self.output_folder_apk):
|
||||
os.mkdir(self.output_folder_apk)
|
||||
|
||||
total_packages = len(self.packages)
|
||||
counter = 0
|
||||
for package in self.packages:
|
||||
for package in packages_selection:
|
||||
counter += 1
|
||||
|
||||
log.info("[%d/%d] Package: %s", counter, total_packages, package.name)
|
||||
log.info("[%d/%d] Package: %s", counter, len(packages_selection),
|
||||
package["package_name"])
|
||||
|
||||
# Get the file path for the specific package.
|
||||
try:
|
||||
output = self._adb_command(f"pm path {package.name}")
|
||||
output = self._clean_output(output)
|
||||
output = self._adb_command(f"pm path {package['package_name']}")
|
||||
output = output.strip().replace("package:", "")
|
||||
if not output:
|
||||
continue
|
||||
except Exception as e:
|
||||
log.exception("Failed to get path of package %s: %s", package.name, e)
|
||||
log.exception("Failed to get path of package %s: %s",
|
||||
package["package_name"], e)
|
||||
self._adb_reconnect()
|
||||
continue
|
||||
|
||||
|
|
@ -188,33 +166,36 @@ class DownloadAPKs(AndroidExtraction):
|
|||
# We loop through each line and download each file.
|
||||
for path in output.split("\n"):
|
||||
device_path = path.strip()
|
||||
file_path = self.pull_package_file(package.name, device_path)
|
||||
file_path = self.pull_package_file(package["package_name"],
|
||||
device_path)
|
||||
if not file_path:
|
||||
continue
|
||||
|
||||
# We add the apk metadata to the package object.
|
||||
package.files.append({
|
||||
file_info = {
|
||||
"path": device_path,
|
||||
"local_name": file_path,
|
||||
"sha256": get_sha256_from_file_path(file_path),
|
||||
})
|
||||
}
|
||||
|
||||
if "files" not in package:
|
||||
package["files"] = [file_info,]
|
||||
else:
|
||||
package["files"].append(file_info)
|
||||
|
||||
log.info("Download of selected packages completed")
|
||||
|
||||
def save_json(self):
|
||||
"""Save the results to the package.json file.
|
||||
"""
|
||||
json_path = os.path.join(self.output_folder, "apks.json")
|
||||
packages = []
|
||||
for package in self.packages:
|
||||
packages.append(package.__dict__)
|
||||
|
||||
with open(json_path, "w") as handle:
|
||||
json.dump(packages, handle, indent=4)
|
||||
json.dump(self.packages, handle, indent=4)
|
||||
|
||||
def run(self):
|
||||
"""Run all steps of fetch-apk.
|
||||
"""
|
||||
self._adb_connect()
|
||||
self.get_packages()
|
||||
self._adb_connect()
|
||||
self.pull_packages()
|
||||
self.save_json()
|
||||
self._adb_disconnect()
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user