diff --git a/mvt/android/cli.py b/mvt/android/cli.py index 0296fd7..77fded2 100644 --- a/mvt/android/cli.py +++ b/mvt/android/cli.py @@ -39,7 +39,7 @@ def cli(): @cli.command("download-apks", help="Download all or non-safelisted installed APKs installed on the device") @click.option("--serial", "-s", type=str, help=HELP_MSG_SERIAL) @click.option("--all-apks", "-a", is_flag=True, - help="Extract all packages installed on the phone, even those marked as safe") + help="Extract all packages installed on the phone, including system packages") @click.option("--virustotal", "-v", is_flag=True, help="Check packages on VirusTotal") @click.option("--koodous", "-k", is_flag=True, help="Check packages on Koodous") @click.option("--all-checks", "-A", is_flag=True, help="Run all available checks") @@ -65,7 +65,8 @@ def download_apks(ctx, all_apks, virustotal, koodous, all_checks, output, from_f log.critical("Unable to create output folder %s: %s", output, e) ctx.exit(1) - download = DownloadAPKs(output_folder=output, all_apks=all_apks) + download = DownloadAPKs(output_folder=output, all_apks=all_apks, + log=logging.getLogger(DownloadAPKs.__module__)) if serial: download.serial = serial download.run() diff --git a/mvt/android/data/safe_packages.txt b/mvt/android/data/safe_packages.txt deleted file mode 100644 index 2898237..0000000 --- a/mvt/android/data/safe_packages.txt +++ /dev/null @@ -1,182 +0,0 @@ -android -android.auto_generated_rro__ -android.autoinstalls.config.google.nexus -com.android.backupconfirm -com.android.bips -com.android.bluetooth -com.android.bluetoothmidiservice -com.android.bookmarkprovider -com.android.calllogbackup -com.android.captiveportallogin -com.android.carrierconfig -com.android.carrierdefaultapp -com.android.cellbroadcastreceiver -com.android.certinstaller -com.android.chrome -com.android.companiondevicemanager -com.android.connectivity.metrics -com.android.cts.ctsshim -com.android.cts.priv.ctsshim -com.android.defcontainer -com.android.documentsui -com.android.dreams.basic -com.android.egg -com.android.emergency -com.android.externalstorage -com.android.facelock -com.android.hotwordenrollment -com.android.hotwordenrollment.okgoogle -com.android.hotwordenrollment.tgoogle -com.android.hotwordenrollment.xgoogle -com.android.htmlviewer -com.android.inputdevices -com.android.keychain -com.android.location.fused -com.android.managedprovisioning -com.android.mms.service -com.android.mtp -com.android.musicfx -com.android.nfc -com.android.omadm.service -com.android.pacprocessor -com.android.phone -com.android.printspooler -com.android.providers.blockednumber -com.android.providers.calendar -com.android.providers.contacts -com.android.providers.downloads -com.android.providers.downloads.ui -com.android.providers.media -com.android.providers.partnerbookmarks -com.android.providers.settings -com.android.providers.telephony -com.android.providers.userdictionary -com.android.proxyhandler -com.android.retaildemo -com.android.safetyregulatoryinfo -com.android.sdm.plugins.connmo -com.android.sdm.plugins.dcmo -com.android.sdm.plugins.diagmon -com.android.sdm.plugins.sprintdm -com.android.server.telecom -com.android.service.ims -com.android.service.ims.presence -com.android.settings -com.android.sharedstoragebackup -com.android.shell -com.android.statementservice -com.android.stk -com.android.systemui -com.android.systemui.theme.dark -com.android.vending -com.android.vpndialogs -com.android.vzwomatrigger -com.android.wallpaperbackup -com.android.wallpaper.livepicker -com.breel.wallpapers -com.customermobile.preload.vzw -com.google.android.apps.cloudprint -com.google.android.apps.docs -com.google.android.apps.docs.editors.docs -com.google.android.apps.enterprise.dmagent -com.google.android.apps.gcs -com.google.android.apps.helprtc -com.google.android.apps.inputmethod.hindi -com.google.android.apps.maps -com.google.android.apps.messaging -com.google.android.apps.nexuslauncher -com.google.android.apps.photos -com.google.android.apps.pixelmigrate -com.google.android.apps.tachyon -com.google.android.apps.turbo -com.google.android.apps.tycho -com.google.android.apps.wallpaper -com.google.android.apps.wallpaper.nexus -com.google.android.apps.work.oobconfig -com.google.android.apps.youtube.vr -com.google.android.asdiv -com.google.android.backuptransport -com.google.android.calculator -com.google.android.calendar -com.google.android.carrier -com.google.android.carrier.authdialog -com.google.android.carrierentitlement -com.google.android.carriersetup -com.google.android.configupdater -com.google.android.contacts -com.google.android.deskclock -com.google.android.dialer -com.google.android.euicc -com.google.android.ext.services -com.google.android.ext.shared -com.google.android.feedback -com.google.android.gm -com.google.android.gms -com.google.android.gms.policy_auth -com.google.android.gms.policy_sidecar_o -com.google.android.gms.setup -com.google.android.GoogleCamera -com.google.android.googlequicksearchbox -com.google.android.gsf -com.google.android.gsf.login -com.google.android.hardwareinfo -com.google.android.hiddenmenu -com.google.android.ims -com.google.android.inputmethod.japanese -com.google.android.inputmethod.korean -com.google.android.inputmethod.latin -com.google.android.inputmethod.pinyin -com.google.android.instantapps.supervisor -com.google.android.keep -com.google.android.marvin.talkback -com.google.android.music -com.google.android.nexusicons -com.google.android.onetimeinitializer -com.google.android.packageinstaller -com.google.android.partnersetup -com.google.android.printservice.recommendation -com.google.android.setupwizard -com.google.android.soundpicker -com.google.android.storagemanager -com.google.android.syncadapters.contacts -com.google.android.tag -com.google.android.talk -com.google.android.tetheringentitlement -com.google.android.theme.pixel -com.google.android.tts -com.google.android.videos -com.google.android.vr.home -com.google.android.vr.inputmethod -com.google.android.webview -com.google.android.wfcactivation -com.google.android.youtube -com.google.ar.core -com.google.intelligence.sense -com.google.modemservice -com.google.pixel.wahoo.gfxdrv -com.google.SSRestartDetector -com.google.tango -com.google.vr.apps.ornament -com.google.vr.vrcore -com.htc.omadm.trigger -com.qti.qualcomm.datastatusnotification -com.qualcomm.atfwd -com.qualcomm.embms -com.qualcomm.fastdormancy -com.qualcomm.ltebc_vzw -com.qualcomm.qcrilmsgtunnel -com.qualcomm.qti.ims -com.qualcomm.qti.networksetting -com.qualcomm.qti.telephonyservice -com.qualcomm.qti.uceShimService -com.qualcomm.shutdownlistner -com.qualcomm.timeservice -com.qualcomm.vzw_api -com.quicinc.cne.CNEService -com.verizon.llkagent -com.verizon.mips.services -com.verizon.obdm -com.verizon.obdm_permissions -com.verizon.services -com.vzw.apnlib -qualcomm.com.vzw_msdc_api diff --git a/mvt/android/download_apks.py b/mvt/android/download_apks.py index b9fc890..28fc42e 100644 --- a/mvt/android/download_apks.py +++ b/mvt/android/download_apks.py @@ -14,6 +14,7 @@ from mvt.common.module import InsufficientPrivileges from mvt.common.utils import get_sha256_from_file_path from .modules.adb.base import AndroidExtraction +from .modules.adb.packages import Packages log = logging.getLogger(__name__) @@ -29,33 +30,23 @@ class PullProgress(tqdm): self.update(current - self.n) -class Package: - """Package indicates a package name and all the files associated with it.""" - - def __init__(self, name, files=None): - self.name = name - self.files = files or [] - - class DownloadAPKs(AndroidExtraction): """DownloadAPKs is the main class operating the download of APKs from the device.""" - def __init__(self, output_folder=None, all_apks=False, packages=None): + def __init__(self, output_folder=None, all_apks=False, log=None, + packages=None): """Initialize module. :param output_folder: Path to the folder where data should be stored :param all_apks: Boolean indicating whether to download all packages or filter known-goods :param packages: Provided list of packages, typically for JSON checks """ - super().__init__(file_path=None, base_folder=None, - output_folder=output_folder) + super().__init__(output_folder=output_folder, log=log) - self.output_folder_apk = None - self.packages = packages or [] + self.packages = packages self.all_apks = all_apks - - self._safe_packages = [] + self.output_folder_apk = None @classmethod def from_json(cls, json_path): @@ -63,55 +54,9 @@ class DownloadAPKs(AndroidExtraction): :param json_path: Path to the apks.json file to parse. """ with open(json_path, "r") as handle: - data = json.load(handle) - - packages = [] - for entry in data: - package = Package(entry["name"], entry["files"]) - packages.append(package) - + packages = json.load(handle) return cls(packages=packages) - def _load_safe_packages(self): - """Load known-good package names. - """ - safe_packages_path = os.path.join("data", "safe_packages.txt") - safe_packages_string = pkg_resources.resource_string(__name__, safe_packages_path) - safe_packages_list = safe_packages_string.decode("utf-8").split("\n") - self._safe_packages.extend(safe_packages_list) - - def _clean_output(self, output): - """Clean adb shell command output. - :param output: Command output to clean. - """ - return output.strip().replace("package:", "") - - def get_packages(self): - """Retrieve package names from the device using adb. - """ - log.info("Retrieving package names ...") - - if not self.all_apks: - self._load_safe_packages() - - output = self._adb_command("pm list packages") - total = 0 - for line in output.split("\n"): - package_name = self._clean_output(line) - if package_name == "": - continue - - total += 1 - - if not self.all_apks and package_name in self._safe_packages: - continue - - if package_name not in self.packages: - self.packages.append(Package(package_name)) - - log.info("There are %d packages installed on the device. I selected %d for inspection.", - total, len(self.packages)) - def pull_package_file(self, package_name, remote_path): """Pull files related to specific package from the device. :param package_name: Name of the package to download @@ -153,6 +98,18 @@ class DownloadAPKs(AndroidExtraction): return local_path + def get_packages(self): + """Use the Packages adb module to retrieve the list of packages. + We reuse the same extraction logic to then download the APKs. + """ + self.log.info("Retrieving list of installed packages...") + + m = Packages() + m.log = self.log + m.run() + + self.packages = m.results + def pull_packages(self): """Download all files of all selected packages from the device. """ @@ -161,26 +118,47 @@ class DownloadAPKs(AndroidExtraction): if not os.path.exists(self.output_folder): os.mkdir(self.output_folder) + # If the user provided the flag --all-apks we select all packages. + packages_selection = [] + if self.all_apks: + log.info("Selected all %d available packages", len(self.packages)) + packages_selection = self.packages + else: + # Otherwise we loop through the packages and get only those that + # are not marked as system. + for package in self.packages: + if not package.get("system", False): + packages_selection.append(package) + + log.info("Selected only %d packages which are not marked as system", + len(packages_selection)) + + if len(packages_selection) == 0: + log.info("No packages were selected for download") + return + log.info("Downloading packages from device. This might take some time ...") self.output_folder_apk = os.path.join(self.output_folder, "apks") if not os.path.exists(self.output_folder_apk): os.mkdir(self.output_folder_apk) - total_packages = len(self.packages) counter = 0 - for package in self.packages: + for package in packages_selection: counter += 1 - log.info("[%d/%d] Package: %s", counter, total_packages, package.name) + log.info("[%d/%d] Package: %s", counter, len(packages_selection), + package["package_name"]) + # Get the file path for the specific package. try: - output = self._adb_command(f"pm path {package.name}") - output = self._clean_output(output) + output = self._adb_command(f"pm path {package['package_name']}") + output = output.strip().replace("package:", "") if not output: continue except Exception as e: - log.exception("Failed to get path of package %s: %s", package.name, e) + log.exception("Failed to get path of package %s: %s", + package["package_name"], e) self._adb_reconnect() continue @@ -188,33 +166,36 @@ class DownloadAPKs(AndroidExtraction): # We loop through each line and download each file. for path in output.split("\n"): device_path = path.strip() - file_path = self.pull_package_file(package.name, device_path) + file_path = self.pull_package_file(package["package_name"], + device_path) if not file_path: continue - # We add the apk metadata to the package object. - package.files.append({ + file_info = { "path": device_path, "local_name": file_path, "sha256": get_sha256_from_file_path(file_path), - }) + } + + if "files" not in package: + package["files"] = [file_info,] + else: + package["files"].append(file_info) + + log.info("Download of selected packages completed") def save_json(self): """Save the results to the package.json file. """ json_path = os.path.join(self.output_folder, "apks.json") - packages = [] - for package in self.packages: - packages.append(package.__dict__) - with open(json_path, "w") as handle: - json.dump(packages, handle, indent=4) + json.dump(self.packages, handle, indent=4) def run(self): """Run all steps of fetch-apk. """ - self._adb_connect() self.get_packages() + self._adb_connect() self.pull_packages() self.save_json() self._adb_disconnect()